542 lines
26 KiB
Plaintext
542 lines
26 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "df1636c8",
|
||
"metadata": {},
|
||
"source": [
|
||
"# Extraction Framework Throughput Test\n",
|
||
"\n",
|
||
"This notepad will test the rate of audio load, preprocessing (resampling, chunking) throughput.\n",
|
||
"\n",
|
||
"All chunked audio will be assigned of embedding `np.zeros(32)` for demonstration purposes."
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 1,
|
||
"id": "9924265b",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/id3.c:process_comment():587] error: No comment text / valid description?\n",
|
||
"[src/libmpg123/id3.c:process_comment():587] error: No comment text / valid description?\n",
|
||
"/home/qt/Repository/DLSiteFSearch/FeatureExtraction/audiopreprocessing.py:26: UserWarning: PySoundFile failed. Trying audioread instead.\n",
|
||
" audio, orig_sr = librosa.load(input_path, sr=None, mono=mono_audio)\n",
|
||
"/home/qt/Repository/DLSiteFSearch/.venv/lib/python3.12/site-packages/librosa/core/audio.py:184: FutureWarning: librosa.core.audio.__audioread_load\n",
|
||
"\tDeprecated as of librosa version 0.10.0.\n",
|
||
"\tIt will be removed in librosa version 1.0.\n",
|
||
" y, sr_native = __audioread_load(path, offset, duration, dtype)\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"/home/qt/Repository/DLSiteFSearch/FeatureExtraction/audiopreprocessing.py:26: UserWarning: PySoundFile failed. Trying audioread instead.\n",
|
||
" audio, orig_sr = librosa.load(input_path, sr=None, mono=mono_audio)\n",
|
||
"/home/qt/Repository/DLSiteFSearch/.venv/lib/python3.12/site-packages/librosa/core/audio.py:184: FutureWarning: librosa.core.audio.__audioread_load\n",
|
||
"\tDeprecated as of librosa version 0.10.0.\n",
|
||
"\tIt will be removed in librosa version 1.0.\n",
|
||
" y, sr_native = __audioread_load(path, offset, duration, dtype)\n",
|
||
"/home/qt/Repository/DLSiteFSearch/FeatureExtraction/audiopreprocessing.py:26: UserWarning: PySoundFile failed. Trying audioread instead.\n",
|
||
" audio, orig_sr = librosa.load(input_path, sr=None, mono=mono_audio)\n",
|
||
"/home/qt/Repository/DLSiteFSearch/.venv/lib/python3.12/site-packages/librosa/core/audio.py:184: FutureWarning: librosa.core.audio.__audioread_load\n",
|
||
"\tDeprecated as of librosa version 0.10.0.\n",
|
||
"\tIt will be removed in librosa version 1.0.\n",
|
||
" y, sr_native = __audioread_load(path, offset, duration, dtype)\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/layer3.c:INT123_do_layer3():1844] error: dequantization failed!\n",
|
||
"[src/libmpg123/id3.c:process_comment():587] error: No comment text / valid description?\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"from dataset_files import MultiThreadedAudioFeatureExtractor, random_audio_chunk\n",
|
||
"import audiopreprocessing\n",
|
||
"import logging\n",
|
||
"logging.basicConfig(format=\"%(asctime)s/%(levelname)s: [%(module)s] %(message)s\", level=logging.INFO)\n",
|
||
"\n",
|
||
"mtafe = MultiThreadedAudioFeatureExtractor(\n",
|
||
" audio_paths=random_audio_chunk(128),\n",
|
||
" max_audio_in_queue=8,\n",
|
||
" audio_feeder_threads=8,\n",
|
||
" feature_extractor_threads=1,\n",
|
||
" desired_sr=32000,\n",
|
||
" force_mono=False,\n",
|
||
" chunk_length=15,\n",
|
||
" chunk_overlap=2\n",
|
||
")\n",
|
||
"audio_map = [audiopreprocessing.load_preprocessed_audio(p, 8000) for p in random_audio_chunk(200)]"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 5,
|
||
"id": "edfa3680",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"[156,\n",
|
||
" 134,\n",
|
||
" 150,\n",
|
||
" 134,\n",
|
||
" 258,\n",
|
||
" 228,\n",
|
||
" 226,\n",
|
||
" 58,\n",
|
||
" 238,\n",
|
||
" 2,\n",
|
||
" 188,\n",
|
||
" 12,\n",
|
||
" 70,\n",
|
||
" 228,\n",
|
||
" 10,\n",
|
||
" 330,\n",
|
||
" 80,\n",
|
||
" 98,\n",
|
||
" 112,\n",
|
||
" 114,\n",
|
||
" 8,\n",
|
||
" 172,\n",
|
||
" 98,\n",
|
||
" 124,\n",
|
||
" 206,\n",
|
||
" 40,\n",
|
||
" 114,\n",
|
||
" 130,\n",
|
||
" 10,\n",
|
||
" 17,\n",
|
||
" 92,\n",
|
||
" 70,\n",
|
||
" 8,\n",
|
||
" 24,\n",
|
||
" 336,\n",
|
||
" 46,\n",
|
||
" 8,\n",
|
||
" 186,\n",
|
||
" 232,\n",
|
||
" 282,\n",
|
||
" 136,\n",
|
||
" 68,\n",
|
||
" 66,\n",
|
||
" 134,\n",
|
||
" 58,\n",
|
||
" 48,\n",
|
||
" 36,\n",
|
||
" 78,\n",
|
||
" 72,\n",
|
||
" 20,\n",
|
||
" 172,\n",
|
||
" 142,\n",
|
||
" 170,\n",
|
||
" 114,\n",
|
||
" 12,\n",
|
||
" 644,\n",
|
||
" 180,\n",
|
||
" 94,\n",
|
||
" 78,\n",
|
||
" 188,\n",
|
||
" 56,\n",
|
||
" 204,\n",
|
||
" 34,\n",
|
||
" 232,\n",
|
||
" 124,\n",
|
||
" 162,\n",
|
||
" 262,\n",
|
||
" 198,\n",
|
||
" 258,\n",
|
||
" 26,\n",
|
||
" 170,\n",
|
||
" 120,\n",
|
||
" 10,\n",
|
||
" 84,\n",
|
||
" 4,\n",
|
||
" 152,\n",
|
||
" 10,\n",
|
||
" 214,\n",
|
||
" 222,\n",
|
||
" 58,\n",
|
||
" 86,\n",
|
||
" 104,\n",
|
||
" 256,\n",
|
||
" 192,\n",
|
||
" 30,\n",
|
||
" 230,\n",
|
||
" 150,\n",
|
||
" 172,\n",
|
||
" 194,\n",
|
||
" 38,\n",
|
||
" 10,\n",
|
||
" 168,\n",
|
||
" 154,\n",
|
||
" 4,\n",
|
||
" 168,\n",
|
||
" 234,\n",
|
||
" 86,\n",
|
||
" 230,\n",
|
||
" 158,\n",
|
||
" 300,\n",
|
||
" 160,\n",
|
||
" 6,\n",
|
||
" 12,\n",
|
||
" 22,\n",
|
||
" 98,\n",
|
||
" 126,\n",
|
||
" 106,\n",
|
||
" 8,\n",
|
||
" 180,\n",
|
||
" 34,\n",
|
||
" 152,\n",
|
||
" 118,\n",
|
||
" 10,\n",
|
||
" 10,\n",
|
||
" 128,\n",
|
||
" 210,\n",
|
||
" 124,\n",
|
||
" 158,\n",
|
||
" 174,\n",
|
||
" 204,\n",
|
||
" 166,\n",
|
||
" 186,\n",
|
||
" 180,\n",
|
||
" 110,\n",
|
||
" 44,\n",
|
||
" 126,\n",
|
||
" 172,\n",
|
||
" 220,\n",
|
||
" 236,\n",
|
||
" 122,\n",
|
||
" 184,\n",
|
||
" 54,\n",
|
||
" 182,\n",
|
||
" 234,\n",
|
||
" 192,\n",
|
||
" 262,\n",
|
||
" 138,\n",
|
||
" 124,\n",
|
||
" 126,\n",
|
||
" 230,\n",
|
||
" 194,\n",
|
||
" 48,\n",
|
||
" 658,\n",
|
||
" 118,\n",
|
||
" 88,\n",
|
||
" 242,\n",
|
||
" 266,\n",
|
||
" 220,\n",
|
||
" 190,\n",
|
||
" 138,\n",
|
||
" 192,\n",
|
||
" 56,\n",
|
||
" 180,\n",
|
||
" 192,\n",
|
||
" 150,\n",
|
||
" 196,\n",
|
||
" 160,\n",
|
||
" 16,\n",
|
||
" 62,\n",
|
||
" 110,\n",
|
||
" 16,\n",
|
||
" 38,\n",
|
||
" 408,\n",
|
||
" 192,\n",
|
||
" 400,\n",
|
||
" 208,\n",
|
||
" 1,\n",
|
||
" 136,\n",
|
||
" 212,\n",
|
||
" 164,\n",
|
||
" 84,\n",
|
||
" 262,\n",
|
||
" 28,\n",
|
||
" 270,\n",
|
||
" 86,\n",
|
||
" 184,\n",
|
||
" 122,\n",
|
||
" 206,\n",
|
||
" 256,\n",
|
||
" 4,\n",
|
||
" 48,\n",
|
||
" 168,\n",
|
||
" 194,\n",
|
||
" 210,\n",
|
||
" 122,\n",
|
||
" 144,\n",
|
||
" 8,\n",
|
||
" 32,\n",
|
||
" 232,\n",
|
||
" 1,\n",
|
||
" 8,\n",
|
||
" 272,\n",
|
||
" 302,\n",
|
||
" 44,\n",
|
||
" 200,\n",
|
||
" 48,\n",
|
||
" 176,\n",
|
||
" 180,\n",
|
||
" 258,\n",
|
||
" 4]"
|
||
]
|
||
},
|
||
"execution_count": 5,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"audio_map_len = [len(m) for m in audio_map]\n",
|
||
"audio_map_len"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "7b2ee365",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"[PosixPath('/mnt/Scratchpad/ASMROne/RJ01312393/03mp3_効果音あり/○ＴＲ２：一回ヤったからって気まずくなるとかｗ.mp3'),\n",
|
||
" PosixPath('/mnt/Scratchpad/ASMROne/RJ01068516/本篇/#7.在漫咖侍寝.wav'),\n",
|
||
" PosixPath('/mnt/MyStuffz/ASMRTwo/RJ01192303/01_舔舐活同伴課程篇/mp3版/06_萌音×梨亞的延長課程.mp3'),\n",
|
||
" PosixPath('/mnt/MyStuffz/ASMRTwo/RJ291279/3_FLAC/アレンジ/04射精音なし.flac'),\n",
|
||
" PosixPath('/mnt/Windows11/ASMRThree/RJ01068246/02：wav/track04_想让你为了姐姐大量射精.wav'),\n",
|
||
" PosixPath('/mnt/Scratchpad/ASMROne/RJ01109943/★萝莉义妹J◯♪豪华特典★/03.兎月りりむ。Freetalk/2-.wav版【最高音质・推荐下载鉴赏】/EX.萝莉义妹@兎月りりむ。Freetalk.wav'),\n",
|
||
" PosixPath('/mnt/Scratchpad/ASMROne/RJ01263522/「游戏迷女友」系列总集篇/边玩游戏边让你自由地使用小穴的游戏迷女友【双声道立体声】/1.音声文件/WAV/2.『顺便帮你舔耳』.wav'),\n",
|
||
" PosixPath('/mnt/Scratchpad/ASMROne/RJ01037597/mp3/07_特典トラック・公募音声.mp3'),\n",
|
||
" PosixPath('/mnt/Scratchpad/ASMROne/RJ271498/mp3/10_【朝フェラ】リラックス朝フェラ.mp3'),\n",
|
||
" PosixPath('/mnt/Scratchpad/ASMROne/RJ01269607/完成音声/mp3-无SE/tr00_报幕.mp3'),\n",
|
||
" PosixPath('/mnt/Scratchpad/ASMROne/RJ304908/mp3/07 酔っ払ったお姉ちゃんとあまあま対面座位セックス.mp3'),\n",
|
||
" PosixPath('/mnt/Windows11/ASMRThree/RJ01153369/02_個別データ/SEなし/08.芦屋もこ/7 挿入⇒喘ぎ.mp3'),\n",
|
||
" PosixPath('/mnt/Scratchpad/ASMROne/RJ01282581/2-.mp3版【圧縮形式・軽量バージョン】/1.SEあり(おすすめ)/6.キスされまくり甘々ピロートーク_SE.mp3'),\n",
|
||
" PosixPath('/mnt/Scratchpad/ASMROne/RJ01070779/wav/06_绀的陪睡（哄睡）.wav'),\n",
|
||
" PosixPath('/mnt/Windows11/ASMRThree/RJ01153369/02_個別データ/SEなし/19.星羅あかね/10 寝息.mp3'),\n",
|
||
" PosixPath('/mnt/Scratchpad/ASMROne/RJ01194525/mp3/TR03_先輩、襲ってもいいんですよ？-.mp3'),\n",
|
||
" PosixPath('/mnt/Windows11/ASMRThree/RJ01167343/1.音声ファイル/環境音無し/7.おまけ『オカズ淫語をたくさん申し上げます♪』.mp3'),\n",
|
||
" PosixPath('/mnt/MyStuffz/ASMRTwo/RJ01205182/wav_no SE/02. 按摩一下耳朵，讓你睡得更舒服～（耳朵按摩）no SE.wav'),\n",
|
||
" PosixPath('/mnt/MyStuffz/ASMRTwo/RJ387999/2.背徳（通常）ルート/2.wav/3.強制愛撫.wav'),\n",
|
||
" PosixPath('/mnt/MyStuffz/ASMRTwo/RJ325846/3_FLAC/ex13【極】脳髄舐め into the abyss.flac'),\n",
|
||
" PosixPath('/mnt/Windows11/ASMRThree/RJ400619/wav/04_PVボイス/フェアリーのキャシー編PV.wav'),\n",
|
||
" PosixPath('/mnt/Windows11/ASMRThree/RJ406548/3-.wav版【ハイレゾ品質・ダウンロード視聴にオススメ】/2.SEなし/04 嘘オホ喘ぎｗｗ＆天使特製オモチャで亀頭いじめ_NoSE.wav'),\n",
|
||
" PosixPath('/mnt/MyStuffz/ASMRTwo/RJ01205182/mp3_no SE/02. 按摩一下耳朵，讓你睡得更舒服～（耳朵按摩）no SE.mp3'),\n",
|
||
" PosixPath('/mnt/Scratchpad/ASMROne/RJ01050049/MP3/03 変態搾精フェラと発情汁まみれオナニー.mp3'),\n",
|
||
" PosixPath('/mnt/Scratchpad/ASMROne/RJ01058640/2-.FLAC版【低損耗形式・推薦線上收聽】/1.SE(推薦)/02.使用性騷擾按摩&戴套性愛來抑制治療小穴性慾♪_SE.flac'),\n",
|
||
" PosixPath('/mnt/MyStuffz/ASMRTwo/RJ01252490/「你要是敢内射我就杀了你！」态度嚣张但尤其敏感的超可爱JK喵子/mp3/Tr.7 附赠音轨.mp3'),\n",
|
||
" PosixPath('/mnt/Windows11/ASMRThree/RJ437868/本編（これがメイン）/限界お漏らしルート/SE一部あり\\u3000お漏らしルート.wav'),\n",
|
||
" PosixPath('/mnt/Windows11/ASMRThree/RJ01123987/1-.mp3版【压缩・适合在线视听】/1.含SE(推荐)/02.去个不停♪连续高潮与肉感紧致小穴性爱♪_SE.mp3'),\n",
|
||
" PosixPath('/mnt/Windows11/ASMRThree/RJ01153369/02_個別データ/SEなし/18.竹早芽衣/2 耳舐め.mp3'),\n",
|
||
" PosixPath('/mnt/Windows11/ASMRThree/RJ437868/ルート別切り抜き/排泄シーンのみ/SEなし\\u3000限界排泄のみ.wav'),\n",
|
||
" PosixPath('/mnt/MyStuffz/ASMRTwo/RJ347971/SE有り/MP3/【8】おまけ：両耳舐めループ.mp3'),\n",
|
||
" PosixPath('/mnt/Scratchpad/ASMROne/RJ01282581/1-.wav版【最高品質・リリムワークス謹製】/1.SEあり(おすすめ)/6.キスされまくり甘々ピロートーク_SE.wav')]"
|
||
]
|
||
},
|
||
"execution_count": 3,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"random_audio_chunk(32)\n",
|
||
"audio_map[0]"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "markdown",
|
||
"id": "f9f16e98",
|
||
"metadata": {},
|
||
"source": [
|
||
"After small scale testing, Three threads: main, `audio_feed` thread and `extractor` thread. Processing 32 files took 60 seconds. After interpolating, processing 9000 files will take approximately 16875 seconds, which is 4,6 hours. It is possible that we need to upgrade both the `audio_feed` and `extractor` into multithreading.\n",
|
||
"\n",
|
||
"After a second round of test.\n",
|
||
"\n",
|
||
"```Processed 200/200 (L:0/W:0 COMPLETE)\n",
|
||
"Extraction completed\n",
|
||
"Took 338.271537993 seconds. Added 27835 vectors/embeddings```\n",
|
||
"\n",
|
||
"The throughput is 200 files in 338 seconds, which is 0.59 files per second. Or 1.69 second per file"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 3,
|
||
"id": "4d453b31",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"8430449"
|
||
]
|
||
},
|
||
"execution_count": 3,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"from dataset_files import serialize_dict_obj\n",
|
||
"from pathlib import Path\n",
|
||
"\n",
|
||
"serialize_dict_obj(Path(\"./testfeature.pkl\").resolve(), afe.features)"
|
||
]
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": ".venv",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.12.3"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|