init

.gitignore

@@ -0,0 +1,160 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:slim
+
+RUN mkdir /app
+
+COPY requirements.txt /app
+
+WORKDIR /app
+
+RUN pip install -r requirements.txt
+
+COPY . .
+
+ENTRYPOINT [ "flask", "run", "-h", "0.0.0.0"]

README.md

@@ -0,0 +1,13 @@
+# Art Emotions Onism Machina
+Web application as a part of Art Object "Onism Machina" [1]
+
+ONISM MACHINA is a project investigating the symbiotic communication between machine and human, exploring the ethical, social and philosophical aspects of the interaction between humans and AI on an emotional level.
+
+### Credits
+- Uses Built-in Webcam of Laptop or Plugged
+- [Human Library](https://github.com/vladmandic/human) - JavaScript library for Face Detection and Recognition
+- [Silero Models](https://github.com/snakers4/silero-models) - The TTS and STT models in PyTorch
+- GhatGPT
+
+### References 
+[1] [Explication](https://docs.google.com/document/d/1iTSPysRRdi9wQZwFZ_Dkf8nqdYims27t-R9DiugFCQ8)

app.py

@@ -0,0 +1,147 @@
+from flask import Flask, render_template, Response, request, jsonify
+import json
+import os
+import torch
+import openai
+
+
+openai.api_key = ""
+
+device = torch.device('cpu')
+torch.set_num_threads(4)
+local_file = 'model.pt'
+
+if not os.path.isfile(local_file):
+    torch.hub.download_url_to_file('https://models.silero.ai/models/tts/ru/v3_1_ru.pt',
+                                   local_file)  
+
+model = torch.package.PackageImporter(local_file).load_pickle("tts_models", "model") # type: ignore
+model.to(device)
+
+sample_rate = 48000
+speaker='xenia'
+example_text = 'В недрах тундры выдры в г+етрах т+ырят в вёдра ядра к+едров.'
+
+state = {}
+state['count'] = 0
+state['size'] = []
+state['gender'] = []
+state['emotion'] = []
+state['age'] = []
+state['prompt'] = ""
+state['need_generation'] = True
+state["new_audio"] = False
+state["generated_text"] = ""
+state["need_audio"] = False
+
+
+app = Flask(__name__)
+# app.logger.setLevel(logging.DEBUG)
+app.logger.info('start logger')
+
+
+@app.route('/send_data', methods=['POST'])
+def send_data():
+    # Получаем данные из запроса
+    data = request.form['data']
+    need_generation = request.form['state']
+    state['need_generation'] = need_generation
+    # Обработка полученных данных
+    detections = json.loads(data)
+    if detections['face']:
+        if state['count'] < 0 or state['new_audio']: state['count'] = 0
+        if state['count'] > 5 and state["need_generation"]:
+            state['count'] = 0
+            emotion = max(set(state['emotion']), key=state['emotion'].count), 
+            sex = max(set(state['gender']), key=state['gender'].count), 
+            age = sum(state['age'])/len(state['age']),
+            app.logger.info(f'{emotion=}, {sex=}, {age=}') 
+            state["prompt"] = generate_prompt(emotion, age, sex)
+            state["generated_text"] = generate_text(state["prompt"]) 
+        elif detections['face'][0]['size'][0] > 200:
+            state['age'].append(detections['face'][0]['age'])
+            state["gender"].append(detections['face'][0]['gender'])
+            state["emotion"].append(detections['face'][0]['emotion'][0]['emotion'])
+            state['count'] += 1
+        else:
+            state['count'] -= 1
+    else:
+        state['count'] -= 1
+        # state["size"].append(detections['face'][0]['size'][0])
+        # print(detections['face'][0])
+    # print(detections['face'][0]['age'], detections['face'][0]['emotion'], detections['face'][0]['gender'])
+
+
+    return data
+
+@app.route('/generate_audio', methods = ["GET", "POST"])
+def generate_audio():
+    app.logger.info('checking need generation')
+
+    if state["need_audio"]:
+        app.logger.info('starting audio generation')
+        audio_paths = model.save_wav(text=state['generated_text'],
+                                    speaker=speaker,
+                                    sample_rate=sample_rate,
+                                    audio_path="static/audio.wav")
+        app.logger.info('generating audio is done')
+        state["new_audio"] = True
+        state["need_generation"] = False
+        state['need_audio'] = False
+    else:
+        state['new_audio'] = False
+        
+    app.logger.info(f'\n{state["need_audio"]=},\n{state["new_audio"]=},\n{state["need_generation"]=}')    
+
+    response = {
+        'newAudio': state["new_audio"],
+        'need_generation': state["need_generation"],
+        'filename': "audio.wav",
+        'text': state['generated_text']
+    }
+
+    return jsonify(response)
+
+@app.route("/audio.wav")
+def audio():
+    # print("Requested path:", request.path)
+    # print("File path:", os.path.join(app.static_folder, 'audio.wav'))
+    return app.send_static_file('audio.wav')
+
+
+@app.route('/')
+def index():
+    """Video streaming home page."""
+    # return render_template('index.html')
+    return render_template('index.html')
+
+
+def generate_prompt(emotion, age, sex):
+    app.logger.info('preload prompt')
+    prompt = f'''Ты — это арт объект выставки про взаимодействие машины и человека. \
+К тебе подходит человек и он показывает эмоцию {emotion}. \
+Ему {age} лет. И это {sex}. \
+Твоя нейросеть распознала эту эмоцию и теперь тебе нужно дать какой-то необычный концептуальный ответ. \
+Что ты скажешь этому человеку?'''
+    return prompt
+
+def generate_text(prompt):
+    app.logger.info("start generating text from openai")
+    response = openai.ChatCompletion.create(
+                        model="gpt-3.5-turbo",
+                        temperature=1,
+                        # max_tokens=1000,
+                        messages=[
+                                {"role": "system", "content": "Ты — это арт объект выставки про взаимодействие машины и человека."},
+                                {"role": "user", "content": prompt},
+                                ])
+    state["need_generation"] = False
+    state["need_audio"] = True
+    app.logger.info("openai generation is done")
+    return response['choices'][0]['message']['content'] # type: ignore
+
+
+if __name__ == '__main__':
+    app.logger.info('start app')
+    app.run(debug=True, host="0.0.0.0") 
+        # ssl_context=("127.0.0.1.pem", "127.0.0.1-key.pem"))

docker-compose.yaml

@@ -0,0 +1,16 @@
+version: '3'
+
+services:
+  symetria_emotions:
+    build: Dockerfile
+    container_name: symetria_emotions
+    restart: always
+    environment:
+      VIRTUAL_HOST: symetria.chad-partners.com
+      LETSENCRYPT_HOST: symetria.chad-partners.com
+    networks:
+      - nginx-proxy
+
+networks:
+  nginx-proxy:
+    external: true

requirements.txt

@@ -0,0 +1,7 @@
+--find-links https://download.pytorch.org/whl/torch_stable.html
+Flask==2.3.2
+openai==0.27.8
+torch=2.0.1+cpu
+torchvision 
+torchaudio 
+# gunicorn

templates/audio.html

@@ -0,0 +1,43 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+    <script src="https://code.jquery.com/jquery-3.7.0.min.js"></script>
+</head>
+
+<body>
+    <audio id="audio-player"></audio>
+    <img src="static/123.jpg">
+    <script>
+        $(document).ready(function () {
+            // Функция для проверки наличия нового аудиофайла на сервере
+            function checkForNewAudio() {
+                $.ajax({
+                    url: '/generate_audio',
+                    method: 'GET',
+                    success: function (response) {
+                        if (response.newAudio) {
+                            // Если есть новый аудиофайл, проигрывайте его на странице
+                            playAudio(response.filename);
+                        }
+                    },
+                    error: function (error) {
+                        console.error('Ошибка при проверке наличия нового аудиофайла:', error);
+                    }
+                });
+            }
+
+            // Функция для проигрывания аудиофайла
+            function playAudio(audioSrc) {
+                const audioPlayer = new Audio(audioSrc);
+                audioPlayer.play();
+            }
+
+            // Периодически проверять наличие нового аудиофайла
+            setInterval(checkForNewAudio, 60000); // Проверять каждые 5 секунд
+        });
+
+    </script>
+</body>
+
+</html>

templates/index.html

@@ -0,0 +1,152 @@
+<!doctype html>
+<html lang="en">
+
+<head>
+    <!-- Required meta tags -->
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <script src="https://cdn.jsdelivr.net/npm/@vladmandic/human/dist/human.js"></script>
+    <script src="https://code.jquery.com/jquery-3.7.0.min.js"></script>
+
+</head>
+
+<body>
+    <div id="data-container"></div>
+    {# <form id="data-form" action="/send_data" method="post"> #}
+        {# <input type="hidden" name="data" id="data-input"> #}
+        {# <input type="submit" value="Отправить данные"> #}
+        <canvas id="canvas" style="margin: 0 auto; width: 100%"></canvas>
+        <pre id="log" style="padding: 8px; position: fixed; bottom: 0"></pre>
+        <script>
+            console.log("start", Human);
+
+
+            const humanConfig = { // user configuration for human, used to fine-tune behavior
+                modelBasePath: 'https://cdn.jsdelivr.net/npm/@vladmandic/human/models/', // models can be loaded directly from cdn as well
+                filter: { enabled: true, equalization: true, flip: false },
+                face: { enabled: true, detector: { rotation: false }, mesh: { enabled: false }, attention: { enabled: true }, iris: { enabled: true }, description: { enabled: true }, emotion: { enabled: true } },
+                body: { enabled: false },
+                hand: { enabled: false },
+                gesture: { enabled: false },
+                object: { enabled: false },
+                segmentation: { enabled: false },
+            };
+
+            const human = new Human.Human(humanConfig);
+            //console.log("continue", human);
+
+            const canvas = document.getElementById('canvas');
+            //const dataForm = document.getElementById('data-form');
+            //const dataInput = document.getElementById('data-input');
+            //const canvas = $('#canvas').get(0)
+
+            var interpolated
+            var need_generation = true
+            var need_playing = true
+            var text
+
+            function splitTextIntoLines(text, wordsPerLine) {
+                const words = text.split(' ');
+                let line = '';
+                let j = 0;
+                for (let i = 0; i < words.length; i++) {
+                    if (j < wordsPerLine) {
+                        line += words[i] + ' '; 
+                        j += 1;
+                    } else {
+                        line += '\n'; 
+                        j = 0;
+                    }
+                }
+                return line;
+            }
+
+            async function drawLoop() { // main screen refresh loop
+                interpolated = human.next(); // get smoothened result using last-known results which are continously updated based on input webcam video
+                human.draw.canvas(human.webcam.element, canvas); // draw webcam video to screen canvas // better than using procesed image as this loop happens faster than processing loop
+                await human.draw.all(canvas, interpolated);
+                document.getElementById('log').innerHTML =
+                    `human version: ${human.version} | ` +
+                    `tfjs version: ${human.tf.version['tfjs-core']}<br>` +
+                    `platform: ${human.env.platform} | ` +
+                    `agent ${human.env.agent}<br>` +
+                    `need_generation ${need_generation}<br>` + // draw labels, boxes, lines, etc.
+                    `text: ${text}`;
+            }
+
+            async function playAudio(audioSrc) {
+                console.log('playing audio')
+                const audioPlayer = new Audio(audioSrc);
+                audioPlayer.addEventListener('ended', function () {
+                    need_generation = true;
+                    need_playing = true;
+                    console.log('playing done');
+                });
+                audioPlayer.play();
+            }
+
+            async function checkForNewAudio() {
+                $.ajax({
+                    url: '/generate_audio',
+                    method: 'GET',
+                    success: function (response) {
+                        need_generation = response.need_generation;
+                        if (response.newAudio && need_playing) {
+                            console.log(response.newAudio)
+                            // Если есть новый аудиофайл, проигрывайте его на странице
+                            text = splitTextIntoLines(response.text, 20);
+                            need_generation = false;
+                            need_playing = false;
+                            playAudio(response.filename);
+                        }
+                    },
+                    error: function (error) {
+                        console.error('Ошибка при проверке наличия нового аудиофайла:', error);
+                    }
+                });
+            }
+
+            async function send_data() {
+                $.ajax({
+                    url: '/send_data',
+                    type: 'POST',
+                    data: { data: JSON.stringify(interpolated), state: need_generation },
+                    success: function (response) {
+                        console.log('face data sent!');
+                    }
+                });
+            };
+
+
+
+            async function main() { // main entry point
+                document.getElementById('log').innerHTML =
+                    `human version: ${human.version} | ` +
+                    `tfjs version: ${human.tf.version['tfjs-core']} <br>` +
+                    `platform: ${human.env.platform} | ` +
+                    `agent ${human.env.agent}<br>` +
+                    `need_generation ${need_generation}<br>` +
+                    `text: ${text}`;
+
+                await human.webcam.start({ crop: true }); // find webcam and start it
+                human.video(human.webcam.element); // instruct human to continously detect video frames
+                canvas.width = human.webcam.width; // set canvas resolution to input webcam native resolution
+                canvas.height = human.webcam.height;
+                canvas.onclick = async () => { // pause when clicked on screen and resume on next click
+                    if (human.webcam.paused) await human.webcam.play();
+                    else human.webcam.pause();
+                };
+                await setInterval(drawLoop, 30); // start draw loop
+                await setInterval(send_data, 2000);
+                await setInterval(checkForNewAudio, 5000)
+            };
+
+
+
+            window.onload = main;
+
+
+        </script>
+</body>
+
+</html>