From 9af5a67530d8c250b75aeeb8bee83b5331151221 Mon Sep 17 00:00:00 2001
From: jb-alvarado <jb@pixelcrusher.de>
Date: Sun, 7 Jun 2020 20:06:14 +0200
Subject: [PATCH 01/17] add ignores

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index f3857de9..822f5bed 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,7 @@
 __pycache__/
 *-orig.*
 *.json
+test/
 tests/
 .pytest_cache/
 venv/

From fbfb9a771229fdf209b5b5c232b93134fec10407 Mon Sep 17 00:00:00 2001
From: jb-alvarado <jb@pixelcrusher.de>
Date: Sun, 7 Jun 2020 20:08:54 +0200
Subject: [PATCH 02/17] modularize output

---
 ffplayout.py                 | 117 +++--------------------------------
 ffplayout.yml                |  27 ++++----
 ffplayout/filters.py         | 107 ++++++++++++++++++++------------
 ffplayout/output/__init__.py |   0
 ffplayout/output/desktop.py  | 104 +++++++++++++++++++++++++++++++
 ffplayout/output/hls.py      |  72 +++++++++++++++++++++
 ffplayout/output/stream.py   | 111 +++++++++++++++++++++++++++++++++
 ffplayout/playlist.py        |   2 +-
 ffplayout/utils.py           |  43 ++++++-------
 9 files changed, 404 insertions(+), 179 deletions(-)
 create mode 100644 ffplayout/output/__init__.py
 create mode 100644 ffplayout/output/desktop.py
 create mode 100644 ffplayout/output/hls.py
 create mode 100644 ffplayout/output/stream.py

diff --git a/ffplayout.py b/ffplayout.py
index 2b1199a9..10111845 100755
--- a/ffplayout.py
+++ b/ffplayout.py
@@ -19,15 +19,9 @@
 # ------------------------------------------------------------------------------
 
 import os
-from subprocess import PIPE, Popen
-from threading import Thread
+from pydoc import locate
 
-from ffplayout.folder import GetSourceFromFolder, MediaStore, MediaWatcher
-from ffplayout.playlist import GetSourceFromPlaylist
-from ffplayout.utils import (_ff, _log, _playlist, _playout, _pre_comp, _text,
-                             ffmpeg_stderr_reader, get_date, messenger,
-                             pre_audio_codec, stdin_args, terminate_processes,
-                             validate_ffmpeg_libs)
+from ffplayout.utils import _playout, validate_ffmpeg_libs
 
 try:
     if os.name != 'posix':
@@ -36,9 +30,6 @@ try:
 except ImportError:
     print('colorama import failed, no colored console output on windows...')
 
-_WINDOWS = os.name == 'nt'
-COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 65424
-
 
 # ------------------------------------------------------------------------------
 # main functions
@@ -49,105 +40,15 @@ def main():
     pipe ffmpeg pre-process to final ffmpeg post-process,
     or play with ffplay
     """
-    year = get_date(False).split('-')[0]
-    overlay = []
 
-    ff_pre_settings = [
-        '-pix_fmt', 'yuv420p', '-r', str(_pre_comp.fps),
-        '-c:v', 'mpeg2video', '-intra',
-        '-b:v', '{}k'.format(_pre_comp.v_bitrate),
-        '-minrate', '{}k'.format(_pre_comp.v_bitrate),
-        '-maxrate', '{}k'.format(_pre_comp.v_bitrate),
-        '-bufsize', '{}k'.format(_pre_comp.v_bufsize)
-        ] + pre_audio_codec() + ['-f', 'mpegts', '-']
+    for output in os.listdir('ffplayout/output'):
+        if os.path.isfile(os.path.join('ffplayout/output', output)) \
+                and output != '__init__.py':
+            mode = os.path.splitext(output)[0]
+            if mode == _playout.mode:
+                output = locate('ffplayout.output.{}.output'.format(mode))
 
-    if _text.add_text:
-        messenger.info('Using drawtext node, listening on address: {}'.format(
-            _text.address
-        ))
-        overlay = [
-            '-vf',
-            "null,zmq=b=tcp\\\\://'{}',drawtext=text='':fontfile='{}'".format(
-                _text.address.replace(':', '\\:'), _text.fontfile)
-        ]
-
-    try:
-        if _playout.preview or stdin_args.desktop:
-            # preview playout to player
-            _ff.encoder = Popen([
-                'ffplay', '-hide_banner', '-nostats', '-i', 'pipe:0'
-                ] + overlay, stderr=PIPE, stdin=PIPE, stdout=None)
-        else:
-            _ff.encoder = Popen([
-                'ffmpeg', '-v', _log.ff_level.lower(), '-hide_banner',
-                '-nostats', '-re', '-thread_queue_size', '256', '-i', 'pipe:0'
-                ] + overlay + [
-                    '-metadata', 'service_name=' + _playout.name,
-                    '-metadata', 'service_provider=' + _playout.provider,
-                    '-metadata', 'year={}'.format(year)
-                ] + _playout.post_comp_param + [_playout.out_addr],
-                stdin=PIPE, stderr=PIPE)
-
-        enc_err_thread = Thread(target=ffmpeg_stderr_reader,
-                                args=(_ff.encoder.stderr, False))
-        enc_err_thread.daemon = True
-        enc_err_thread.start()
-
-        if _playlist.mode and not stdin_args.folder:
-            watcher = None
-            get_source = GetSourceFromPlaylist()
-        else:
-            messenger.info('Start folder mode')
-            media = MediaStore()
-            watcher = MediaWatcher(media)
-            get_source = GetSourceFromFolder(media)
-
-        try:
-            for src_cmd in get_source.next():
-                messenger.debug('src_cmd: "{}"'.format(src_cmd))
-                if src_cmd[0] == '-i':
-                    current_file = src_cmd[1]
-                else:
-                    current_file = src_cmd[3]
-
-                messenger.info('Play: "{}"'.format(current_file))
-
-                with Popen([
-                    'ffmpeg', '-v', _log.ff_level.lower(), '-hide_banner',
-                    '-nostats'] + src_cmd + ff_pre_settings,
-                        stdout=PIPE, stderr=PIPE) as _ff.decoder:
-
-                    dec_err_thread = Thread(target=ffmpeg_stderr_reader,
-                                            args=(_ff.decoder.stderr, True))
-                    dec_err_thread.daemon = True
-                    dec_err_thread.start()
-
-                    while True:
-                        buf = _ff.decoder.stdout.read(COPY_BUFSIZE)
-                        if not buf:
-                            break
-                        _ff.encoder.stdin.write(buf)
-
-        except BrokenPipeError:
-            messenger.error('Broken Pipe!')
-            terminate_processes(watcher)
-
-        except SystemExit:
-            messenger.info('Got close command')
-            terminate_processes(watcher)
-
-        except KeyboardInterrupt:
-            messenger.warning('Program terminated')
-            terminate_processes(watcher)
-
-        # close encoder when nothing is to do anymore
-        if _ff.encoder.poll() is None:
-            _ff.encoder.terminate()
-
-    finally:
-        if _ff.encoder.poll() is None:
-            _ff.encoder.terminate()
-        _ff.encoder.wait()
+                output()
 
 
 if __name__ == '__main__':
diff --git a/ffplayout.yml b/ffplayout.yml
index b8227814..aaf5f0cf 100644
--- a/ffplayout.yml
+++ b/ffplayout.yml
@@ -31,16 +31,18 @@ logging:
     log_level: "DEBUG"
     ffmpeg_level: "ERROR"
 
-pre_compress:
-    helptext: Settings for the pre-compression. All clips get prepared in that way,
+pre_process:
+    helptext: Settings for the pre_process. All clips get prepared in that way,
         so the input for the final compression is unique. 'aspect' must be a float
         number. 'logo' is only used if the path exist. 'logo_scale' scale the logo to
         target size, leave it blank when no scaling is needed, format is 'number:number',
         for example '100:-1' for proportional scaling. With 'logo_opacity' logo can
         become transparent. With 'logo_filter' 'overlay=W-w-12:12' you can modify
         the logo position. With 'use_loudnorm' you can activate single pass EBU R128
-        loudness normalization. 'loud_*' can adjust the loudnorm filter. [Output is
-        always progressive!]
+        loudness normalization. 'loud_*' can adjust the loudnorm filter. 'output_count'
+        sets the outputs for the filtering, > 1 gives the option to use the same filters
+        for multiple outputs. This outputs can be taken in 'ffmpeg_param', names will be
+        vout2, vout3; aout2, aout2 etc.
     width: 1024
     height: 576
     aspect: 1.778
@@ -54,6 +56,7 @@ pre_compress:
     loud_I: -18
     loud_TP: -1.5
     loud_LRA: 11
+    output_count: 1
 
 playlist:
     helptext: Set 'playlist_mode' to 'False' if you want to play clips from the [STORAGE]
@@ -84,18 +87,21 @@ text:
     helptext: Overlay text in combination with libzmq for remote text manipulation.
         On windows fontfile path need to be like this 'C\:/WINDOWS/fonts/DejaVuSans.ttf'.
         In a standard environment the filter drawtext node is Parsed_drawtext_2.
+        'over_pre' if True text will be overlay in pre processing. Continue same text
+        over multiple files is in that mode not possible.
     add_text: False
+    over_pre: False
     bind_address: "127.0.0.1:5555"
     fontfile: "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"
 
 out:
-    helptext: The final playout post compression. Set the settings to your needs.
-        'preview' works only on a desktop system with ffplay!! Set it to 'True', if
-        you need it.
-    preview: False
+    helptext: The final playout compression. Set the settings to your needs.
+        'mode' has the standard options 'desktop', 'hls', 'stream'. Self made outputs
+        can be define, by adding script in output folder with an 'output' function inside.
+    mode: 'stream'
     service_name: "Live Stream"
     service_provider: "example.org"
-    post_ffmpeg_param: >-
+    ffmpeg_param: >-
         -c:v libx264
         -crf 23
         -x264-params keyint=50:min-keyint=25:scenecut=-1
@@ -108,5 +114,4 @@ out:
         -ar 44100
         -b:a 128k
         -flags +global_header
-        -f flv
-    out_addr: "rtmp://localhost/live/stream"
+        -f flv "rtmp://localhost/live/stream"
diff --git a/ffplayout/filters.py b/ffplayout/filters.py
index 3a93a97a..1d057f90 100644
--- a/ffplayout/filters.py
+++ b/ffplayout/filters.py
@@ -21,13 +21,25 @@ import math
 import os
 import re
 
-from .utils import _pre_comp
+from .utils import _pre, _text
 
 # ------------------------------------------------------------------------------
 # building filters,
 # when is needed add individuell filters to match output format
 # ------------------------------------------------------------------------------
 
+
+def text_filter():
+    filter_chain = []
+
+    if _text.add_text and _text.over_pre:
+        filter_chain = [
+            "null,zmq=b=tcp\\\\://'{}',drawtext=text='':fontfile='{}'".format(
+                _text.address.replace(':', '\\:'), _text.fontfile)]
+
+    return filter_chain
+
+
 def deinterlace_filter(probe):
     """
     when material is interlaced,
@@ -50,15 +62,15 @@ def pad_filter(probe):
     filter_chain = []
 
     if not math.isclose(probe.video[0]['aspect'],
-                        _pre_comp.aspect, abs_tol=0.03):
-        if probe.video[0]['aspect'] < _pre_comp.aspect:
+                        _pre.aspect, abs_tol=0.03):
+        if probe.video[0]['aspect'] < _pre.aspect:
             filter_chain.append(
-                'pad=ih*{}/{}/sar:ih:(ow-iw)/2:(oh-ih)/2'.format(_pre_comp.w,
-                                                                 _pre_comp.h))
-        elif probe.video[0]['aspect'] > _pre_comp.aspect:
+                'pad=ih*{}/{}/sar:ih:(ow-iw)/2:(oh-ih)/2'.format(_pre.w,
+                                                                 _pre.h))
+        elif probe.video[0]['aspect'] > _pre.aspect:
             filter_chain.append(
-                'pad=iw:iw*{}/{}/sar:(ow-iw)/2:(oh-ih)/2'.format(_pre_comp.h,
-                                                                 _pre_comp.w))
+                'pad=iw:iw*{}/{}/sar:(ow-iw)/2:(oh-ih)/2'.format(_pre.h,
+                                                                 _pre.w))
 
     return filter_chain
 
@@ -69,8 +81,8 @@ def fps_filter(probe):
     """
     filter_chain = []
 
-    if probe.video[0]['fps'] != _pre_comp.fps:
-        filter_chain.append('fps={}'.format(_pre_comp.fps))
+    if probe.video[0]['fps'] != _pre.fps:
+        filter_chain.append('fps={}'.format(_pre.fps))
 
     return filter_chain
 
@@ -82,13 +94,13 @@ def scale_filter(probe):
     """
     filter_chain = []
 
-    if int(probe.video[0]['width']) != _pre_comp.w or \
-            int(probe.video[0]['height']) != _pre_comp.h:
-        filter_chain.append('scale={}:{}'.format(_pre_comp.w, _pre_comp.h))
+    if int(probe.video[0]['width']) != _pre.w or \
+            int(probe.video[0]['height']) != _pre.h:
+        filter_chain.append('scale={}:{}'.format(_pre.w, _pre.h))
 
     if not math.isclose(probe.video[0]['aspect'],
-                        _pre_comp.aspect, abs_tol=0.03):
-        filter_chain.append('setdar=dar={}'.format(_pre_comp.aspect))
+                        _pre.aspect, abs_tol=0.03):
+        filter_chain.append('setdar=dar={}'.format(_pre.aspect))
 
     return filter_chain
 
@@ -115,27 +127,27 @@ def overlay_filter(duration, ad, ad_last, ad_next):
     when ad is comming next fade logo out,
     when clip before was an ad fade logo in
     """
-    logo_filter = '[v]null[logo]'
+    logo_filter = '[v]null'
     scale_filter = ''
 
-    if _pre_comp.add_logo and os.path.isfile(_pre_comp.logo) and not ad:
+    if _pre.add_logo and os.path.isfile(_pre.logo) and not ad:
         logo_chain = []
-        if _pre_comp.logo_scale and \
-                re.match(r'\d+:-?\d+', _pre_comp.logo_scale):
-            scale_filter = 'scale={},'.format(_pre_comp.logo_scale)
+        if _pre.logo_scale and \
+                re.match(r'\d+:-?\d+', _pre.logo_scale):
+            scale_filter = 'scale={},'.format(_pre.logo_scale)
         logo_extras = 'format=rgba,{}colorchannelmixer=aa={}'.format(
-            scale_filter, _pre_comp.logo_opacity)
+            scale_filter, _pre.logo_opacity)
         loop = 'loop=loop=-1:size=1:start=0'
         logo_chain.append(
-            'movie={},{},{}'.format(_pre_comp.logo, loop, logo_extras))
+            'movie={},{},{}'.format(_pre.logo, loop, logo_extras))
         if ad_last:
             logo_chain.append('fade=in:st=0:d=1.0:alpha=1')
         if ad_next:
             logo_chain.append('fade=out:st={}:d=1.0:alpha=1'.format(
                 duration - 1))
 
-        logo_filter = '{}[l];[v][l]{}:shortest=1[logo]'.format(
-            ','.join(logo_chain), _pre_comp.logo_filter)
+        logo_filter = '{}[l];[v][l]{}:shortest=1'.format(
+            ','.join(logo_chain), _pre.logo_filter)
 
     return logo_filter
 
@@ -161,9 +173,9 @@ def add_loudnorm(probe):
     """
     loud_filter = []
 
-    if probe.audio and _pre_comp.add_loudnorm:
+    if probe.audio and _pre.add_loudnorm:
         loud_filter = [('loudnorm=I={}:TP={}:LRA={}').format(
-            _pre_comp.loud_i, _pre_comp.loud_tp, _pre_comp.loud_lra)]
+            _pre.loud_i, _pre.loud_tp, _pre.loud_lra)]
 
     return loud_filter
 
@@ -175,7 +187,7 @@ def extend_audio(probe, duration):
     pad_filter = []
 
     if probe.audio and 'duration' in probe.audio[0] and \
-            duration > float(probe.audio[0]['duration']) + 0.3:
+            duration > float(probe.audio[0]['duration']) + 0.1:
         pad_filter.append('apad=whole_dur={}'.format(duration))
 
     return pad_filter
@@ -189,25 +201,45 @@ def extend_video(probe, duration, target_duration):
 
     if 'duration' in probe.video[0] and \
         target_duration < duration > float(
-            probe.video[0]['duration']) + 0.3:
+            probe.video[0]['duration']) + 0.1:
         pad_filter.append('tpad=stop_mode=add:stop_duration={}'.format(
             duration - float(probe.video[0]['duration'])))
 
     return pad_filter
 
 
+def split_filter(filter_type):
+    map_node = []
+    filter_prefix = ''
+
+    if filter_type == 'a':
+        filter_prefix = 'a'
+
+    if _pre.output_count > 1:
+        for num in range(_pre.output_count):
+            map_node.append('[{}out{}]'.format(filter_type, num + 1))
+
+        filter = ',{}split={}{}'.format(filter_prefix, _pre.output_count,
+                                        ''.join(map_node))
+
+    else:
+        filter = '[{}out1]'.format(filter_type)
+
+    return filter
+
+
 def build_filtergraph(duration, seek, out, ad, ad_last, ad_next, probe, msg):
     """
     build final filter graph, with video and audio chain
     """
     video_chain = []
     audio_chain = []
-    video_map = ['-map', '[logo]']
 
     if out > duration:
         seek = 0
 
     if probe.video[0]:
+        video_chain += text_filter()
         video_chain += deinterlace_filter(probe)
         video_chain += pad_filter(probe)
         video_chain += fps_filter(probe)
@@ -229,17 +261,16 @@ def build_filtergraph(duration, seek, out, ad, ad_last, ad_next, probe, msg):
         video_filter = 'null[v]'
 
     logo_filter = overlay_filter(out - seek, ad, ad_last, ad_next)
+    v_split = split_filter('v')
+    video_map = ['-map', '[vout1]']
     video_filter = [
-        '-filter_complex', '[0:v]{};{}'.format(
-            video_filter, logo_filter)]
+        '-filter_complex', '[0:v]{};{}{}'.format(
+            video_filter, logo_filter, v_split)]
 
-    if audio_chain:
-        audio_filter = [
-            '-filter_complex', '{}[a]'.format(','.join(audio_chain))]
-        audio_map = ['-map', '[a]']
-    else:
-        audio_filter = []
-        audio_map = ['-map', '0:a']
+    a_split = split_filter('a')
+    audio_map = ['-map', '[aout1]']
+    audio_filter = [
+        '-filter_complex', '{}{}'.format(','.join(audio_chain), a_split)]
 
     if probe.video[0]:
         return video_filter + audio_filter + video_map + audio_map
diff --git a/ffplayout/output/__init__.py b/ffplayout/output/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/ffplayout/output/desktop.py b/ffplayout/output/desktop.py
new file mode 100644
index 00000000..efdc7f34
--- /dev/null
+++ b/ffplayout/output/desktop.py
@@ -0,0 +1,104 @@
+import os
+from subprocess import PIPE, Popen
+from threading import Thread
+
+from ffplayout.folder import GetSourceFromFolder, MediaStore, MediaWatcher
+from ffplayout.playlist import GetSourceFromPlaylist
+from ffplayout.utils import (_ff, _log, _playlist, _pre, _text,
+                             ffmpeg_stderr_reader, messenger, pre_audio_codec,
+                             stdin_args, terminate_processes)
+
+_WINDOWS = os.name == 'nt'
+COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 65424
+
+
+def output():
+    """
+    this output is for playing on desktop with ffplay
+    """
+    overlay = []
+
+    ff_pre_settings = [
+        '-pix_fmt', 'yuv420p', '-r', str(_pre.fps),
+        '-c:v', 'mpeg2video', '-intra',
+        '-b:v', '{}k'.format(_pre.v_bitrate),
+        '-minrate', '{}k'.format(_pre.v_bitrate),
+        '-maxrate', '{}k'.format(_pre.v_bitrate),
+        '-bufsize', '{}k'.format(_pre.v_bufsize)
+        ] + pre_audio_codec() + ['-f', 'mpegts', '-']
+
+    if _text.add_text and not _text.over_pre:
+        messenger.info('Using drawtext node, listening on address: {}'.format(
+            _text.address
+        ))
+        overlay = [
+            '-vf',
+            "null,zmq=b=tcp\\\\://'{}',drawtext=text='':fontfile='{}'".format(
+                _text.address.replace(':', '\\:'), _text.fontfile)
+        ]
+
+    try:
+        _ff.encoder = Popen([
+            'ffplay', '-hide_banner', '-nostats', '-i', 'pipe:0'
+            ] + overlay, stderr=PIPE, stdin=PIPE, stdout=None)
+
+        enc_err_thread = Thread(target=ffmpeg_stderr_reader,
+                                args=(_ff.encoder.stderr, False))
+        enc_err_thread.daemon = True
+        enc_err_thread.start()
+
+        if _playlist.mode and not stdin_args.folder:
+            watcher = None
+            get_source = GetSourceFromPlaylist()
+        else:
+            messenger.info('Start folder mode')
+            media = MediaStore()
+            watcher = MediaWatcher(media)
+            get_source = GetSourceFromFolder(media)
+
+        try:
+            for src_cmd in get_source.next():
+                messenger.debug('src_cmd: "{}"'.format(src_cmd))
+                if src_cmd[0] == '-i':
+                    current_file = src_cmd[1]
+                else:
+                    current_file = src_cmd[3]
+
+                messenger.info('Play: "{}"'.format(current_file))
+
+                with Popen([
+                    'ffmpeg', '-v', _log.ff_level.lower(), '-hide_banner',
+                    '-nostats'] + src_cmd + ff_pre_settings,
+                        stdout=PIPE, stderr=PIPE) as _ff.decoder:
+
+                    dec_err_thread = Thread(target=ffmpeg_stderr_reader,
+                                            args=(_ff.decoder.stderr, True))
+                    dec_err_thread.daemon = True
+                    dec_err_thread.start()
+
+                    while True:
+                        buf = _ff.decoder.stdout.read(COPY_BUFSIZE)
+                        if not buf:
+                            break
+                        _ff.encoder.stdin.write(buf)
+
+        except BrokenPipeError:
+            messenger.error('Broken Pipe!')
+            terminate_processes(watcher)
+
+        except SystemExit:
+            messenger.info('Got close command')
+            terminate_processes(watcher)
+
+        except KeyboardInterrupt:
+            messenger.warning('Program terminated')
+            terminate_processes(watcher)
+
+        # close encoder when nothing is to do anymore
+        if _ff.encoder.poll() is None:
+            _ff.encoder.terminate()
+
+    finally:
+        if _ff.encoder.poll() is None:
+            _ff.encoder.terminate()
+        _ff.encoder.wait()
diff --git a/ffplayout/output/hls.py b/ffplayout/output/hls.py
new file mode 100644
index 00000000..7efe4eba
--- /dev/null
+++ b/ffplayout/output/hls.py
@@ -0,0 +1,72 @@
+from subprocess import PIPE, Popen
+from threading import Thread
+
+from ffplayout.folder import GetSourceFromFolder, MediaStore, MediaWatcher
+from ffplayout.playlist import GetSourceFromPlaylist
+from ffplayout.utils import (_ff, _log, _playlist, _playout,
+                             ffmpeg_stderr_reader, get_date, messenger,
+                             stdin_args, terminate_processes)
+
+
+def output():
+    """
+    this output is hls output, no preprocess is needed.
+    """
+    year = get_date(False).split('-')[0]
+
+    try:
+        if _playlist.mode and not stdin_args.folder:
+            watcher = None
+            get_source = GetSourceFromPlaylist()
+        else:
+            messenger.info('Start folder mode')
+            media = MediaStore()
+            watcher = MediaWatcher(media)
+            get_source = GetSourceFromFolder(media)
+
+        try:
+            for src_cmd in get_source.next():
+                messenger.debug('src_cmd: "{}"'.format(src_cmd))
+                if src_cmd[0] == '-i':
+                    current_file = src_cmd[1]
+                else:
+                    current_file = src_cmd[3]
+
+                messenger.info('Play: "{}"'.format(current_file))
+                cmd = [
+                    'ffmpeg', '-v', _log.ff_level.lower(), '-hide_banner',
+                    '-nostats', '-re', '-thread_queue_size', '256'
+                    ] + src_cmd + [
+                        '-metadata', 'service_name=' + _playout.name,
+                        '-metadata', 'service_provider=' + _playout.provider,
+                        '-metadata', 'year={}'.format(year)
+                    ] + _playout.ffmpeg_param
+
+                _ff.encoder = Popen(cmd, stdin=PIPE, stderr=PIPE)
+
+                enc_thread = Thread(target=ffmpeg_stderr_reader,
+                                    args=(_ff.encoder.stderr, True))
+                enc_thread.daemon = True
+                enc_thread.start()
+                enc_thread.join()
+
+        except BrokenPipeError:
+            messenger.error('Broken Pipe!')
+            terminate_processes(watcher)
+
+        except SystemExit:
+            messenger.info('Got close command')
+            terminate_processes(watcher)
+
+        except KeyboardInterrupt:
+            messenger.warning('Program terminated')
+            terminate_processes(watcher)
+
+        # close encoder when nothing is to do anymore
+        if _ff.encoder.poll() is None:
+            _ff.encoder.terminate()
+
+    finally:
+        if _ff.encoder.poll() is None:
+            _ff.encoder.terminate()
+        _ff.encoder.wait()
diff --git a/ffplayout/output/stream.py b/ffplayout/output/stream.py
new file mode 100644
index 00000000..200aa1dc
--- /dev/null
+++ b/ffplayout/output/stream.py
@@ -0,0 +1,111 @@
+import os
+from subprocess import PIPE, Popen
+from threading import Thread
+
+from ffplayout.folder import GetSourceFromFolder, MediaStore, MediaWatcher
+from ffplayout.playlist import GetSourceFromPlaylist
+from ffplayout.utils import (_ff, _log, _playlist, _playout, _pre, _text,
+                             ffmpeg_stderr_reader, get_date, messenger,
+                             pre_audio_codec, stdin_args, terminate_processes)
+
+_WINDOWS = os.name == 'nt'
+COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 65424
+
+
+def output():
+    """
+    this output is for streaming to a target address,
+    like rtmp, rtp, svt, etc.
+    """
+    year = get_date(False).split('-')[0]
+    overlay = []
+
+    ff_pre_settings = [
+        '-pix_fmt', 'yuv420p', '-r', str(_pre.fps),
+        '-c:v', 'mpeg2video', '-intra',
+        '-b:v', '{}k'.format(_pre.v_bitrate),
+        '-minrate', '{}k'.format(_pre.v_bitrate),
+        '-maxrate', '{}k'.format(_pre.v_bitrate),
+        '-bufsize', '{}k'.format(_pre.v_bufsize)
+        ] + pre_audio_codec() + ['-f', 'mpegts', '-']
+
+    if _text.add_text and not _text.over_pre:
+        messenger.info('Using drawtext node, listening on address: {}'.format(
+            _text.address
+        ))
+        overlay = [
+            '-vf',
+            "null,zmq=b=tcp\\\\://'{}',drawtext=text='':fontfile='{}'".format(
+                _text.address.replace(':', '\\:'), _text.fontfile)
+        ]
+
+    try:
+        _ff.encoder = Popen([
+            'ffmpeg', '-v', _log.ff_level.lower(), '-hide_banner',
+            '-nostats', '-re', '-thread_queue_size', '256', '-i', 'pipe:0'
+            ] + overlay + [
+                '-metadata', 'service_name=' + _playout.name,
+                '-metadata', 'service_provider=' + _playout.provider,
+                '-metadata', 'year={}'.format(year)
+            ] + _playout.ffmpeg_param, stdin=PIPE, stderr=PIPE)
+
+        enc_err_thread = Thread(target=ffmpeg_stderr_reader,
+                                args=(_ff.encoder.stderr, False))
+        enc_err_thread.daemon = True
+        enc_err_thread.start()
+
+        if _playlist.mode and not stdin_args.folder:
+            watcher = None
+            get_source = GetSourceFromPlaylist()
+        else:
+            messenger.info('Start folder mode')
+            media = MediaStore()
+            watcher = MediaWatcher(media)
+            get_source = GetSourceFromFolder(media)
+
+        try:
+            for src_cmd in get_source.next():
+                messenger.debug('src_cmd: "{}"'.format(src_cmd))
+                if src_cmd[0] == '-i':
+                    current_file = src_cmd[1]
+                else:
+                    current_file = src_cmd[3]
+
+                messenger.info('Play: "{}"'.format(current_file))
+
+                with Popen([
+                    'ffmpeg', '-v', _log.ff_level.lower(), '-hide_banner',
+                    '-nostats'] + src_cmd + ff_pre_settings,
+                        stdout=PIPE, stderr=PIPE) as _ff.decoder:
+
+                    dec_err_thread = Thread(target=ffmpeg_stderr_reader,
+                                            args=(_ff.decoder.stderr, True))
+                    dec_err_thread.daemon = True
+                    dec_err_thread.start()
+
+                    while True:
+                        buf = _ff.decoder.stdout.read(COPY_BUFSIZE)
+                        if not buf:
+                            break
+                        _ff.encoder.stdin.write(buf)
+
+        except BrokenPipeError:
+            messenger.error('Broken Pipe!')
+            terminate_processes(watcher)
+
+        except SystemExit:
+            messenger.info('Got close command')
+            terminate_processes(watcher)
+
+        except KeyboardInterrupt:
+            messenger.warning('Program terminated')
+            terminate_processes(watcher)
+
+        # close encoder when nothing is to do anymore
+        if _ff.encoder.poll() is None:
+            _ff.encoder.terminate()
+
+    finally:
+        if _ff.encoder.poll() is None:
+            _ff.encoder.terminate()
+        _ff.encoder.wait()
diff --git a/ffplayout/playlist.py b/ffplayout/playlist.py
index cec5dca2..f666ace4 100644
--- a/ffplayout/playlist.py
+++ b/ffplayout/playlist.py
@@ -224,7 +224,7 @@ class GetSourceFromPlaylist:
 
             if self.clip_nodes is None:
                 self.eof_handling(
-                    'No valid playlist:\n{}'.format(self.json_file), True, 300)
+                    'No valid playlist:\n{}'.format(self.json_file), True, 30)
                 yield self.src_cmd + self.filtergraph
                 continue
 
diff --git a/ffplayout/utils.py b/ffplayout/utils.py
index b737b9a0..143bedcc 100644
--- a/ffplayout/utils.py
+++ b/ffplayout/utils.py
@@ -113,7 +113,7 @@ def get_time(time_format):
 _general = SimpleNamespace()
 _mail = SimpleNamespace()
 _log = SimpleNamespace()
-_pre_comp = SimpleNamespace()
+_pre = SimpleNamespace()
 _playlist = SimpleNamespace()
 _storage = SimpleNamespace()
 _text = SimpleNamespace()
@@ -178,15 +178,16 @@ def load_config():
     _mail.recip = cfg['mail']['recipient']
     _mail.level = cfg['mail']['mail_level']
 
-    _pre_comp.add_logo = cfg['pre_compress']['add_logo']
-    _pre_comp.logo = cfg['pre_compress']['logo']
-    _pre_comp.logo_scale = cfg['pre_compress']['logo_scale']
-    _pre_comp.logo_filter = cfg['pre_compress']['logo_filter']
-    _pre_comp.logo_opacity = cfg['pre_compress']['logo_opacity']
-    _pre_comp.add_loudnorm = cfg['pre_compress']['add_loudnorm']
-    _pre_comp.loud_i = cfg['pre_compress']['loud_I']
-    _pre_comp.loud_tp = cfg['pre_compress']['loud_TP']
-    _pre_comp.loud_lra = cfg['pre_compress']['loud_LRA']
+    _pre.add_logo = cfg['pre_process']['add_logo']
+    _pre.logo = cfg['pre_process']['logo']
+    _pre.logo_scale = cfg['pre_process']['logo_scale']
+    _pre.logo_filter = cfg['pre_process']['logo_filter']
+    _pre.logo_opacity = cfg['pre_process']['logo_opacity']
+    _pre.add_loudnorm = cfg['pre_process']['add_loudnorm']
+    _pre.loud_i = cfg['pre_process']['loud_I']
+    _pre.loud_tp = cfg['pre_process']['loud_TP']
+    _pre.loud_lra = cfg['pre_process']['loud_LRA']
+    _pre.output_count = cfg['pre_process']['output_count']
 
     _playlist.mode = cfg['playlist']['playlist_mode']
     _playlist.path = cfg['playlist']['path']
@@ -199,6 +200,7 @@ def load_config():
     _storage.shuffle = cfg['storage']['shuffle']
 
     _text.add_text = cfg['text']['add_text']
+    _text.over_pre = cfg['text']['over_pre']
     _text.address = cfg['text']['bind_address']
     _text.fontfile = cfg['text']['fontfile']
 
@@ -209,18 +211,17 @@ def load_config():
         _log.level = cfg['logging']['log_level']
         _log.ff_level = cfg['logging']['ffmpeg_level']
 
-        _pre_comp.w = cfg['pre_compress']['width']
-        _pre_comp.h = cfg['pre_compress']['height']
-        _pre_comp.aspect = cfg['pre_compress']['aspect']
-        _pre_comp.fps = cfg['pre_compress']['fps']
-        _pre_comp.v_bitrate = cfg['pre_compress']['width'] * 50
-        _pre_comp.v_bufsize = cfg['pre_compress']['width'] * 50 / 2
+        _pre.w = cfg['pre_process']['width']
+        _pre.h = cfg['pre_process']['height']
+        _pre.aspect = cfg['pre_process']['aspect']
+        _pre.fps = cfg['pre_process']['fps']
+        _pre.v_bitrate = cfg['pre_process']['width'] * 50
+        _pre.v_bufsize = cfg['pre_process']['width'] * 50 / 2
 
-        _playout.preview = cfg['out']['preview']
+        _playout.mode = cfg['out']['mode']
         _playout.name = cfg['out']['service_name']
         _playout.provider = cfg['out']['service_provider']
-        _playout.post_comp_param = cfg['out']['post_ffmpeg_param'].split(' ')
-        _playout.out_addr = cfg['out']['out_addr']
+        _playout.ffmpeg_param = cfg['out']['ffmpeg_param'].split(' ')
 
         _init.load = False
 
@@ -782,7 +783,7 @@ def gen_dummy(duration):
     return [
         '-f', 'lavfi', '-i',
         'color=c={}:s={}x{}:d={}:r={},format=pix_fmts=yuv420p'.format(
-            color, _pre_comp.w, _pre_comp.h, duration, _pre_comp.fps
+            color, _pre.w, _pre.h, duration, _pre.fps
         ),
         '-f', 'lavfi', '-i', 'anoisesrc=d={}:c=pink:r=48000:a=0.05'.format(
             duration)
@@ -989,7 +990,7 @@ def pre_audio_codec():
     s302m has higher quality, but is experimental
     and works not well together with the loudnorm filter
     """
-    if _pre_comp.add_loudnorm:
+    if _pre.add_loudnorm:
         acodec = 'libtwolame' if 'libtwolame' in FF_LIBS['libs'] else 'mp2'
         return ['-c:a', acodec, '-b:a', '384k', '-ar', '48000', '-ac', '2']
     else:

From 64d9dacd756808ff26268ed28b219f5deae3b3b5 Mon Sep 17 00:00:00 2001
From: jb-alvarado <jb@pixelcrusher.de>
Date: Sun, 28 Jun 2020 21:48:49 +0200
Subject: [PATCH 03/17] change example filter output

---
 ffplayout.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ffplayout.yml b/ffplayout.yml
index aaf5f0cf..6563dcc9 100644
--- a/ffplayout.yml
+++ b/ffplayout.yml
@@ -42,7 +42,7 @@ pre_process:
         loudness normalization. 'loud_*' can adjust the loudnorm filter. 'output_count'
         sets the outputs for the filtering, > 1 gives the option to use the same filters
         for multiple outputs. This outputs can be taken in 'ffmpeg_param', names will be
-        vout2, vout3; aout2, aout2 etc.
+        vout1, vout2; aout1, aout2 etc.
     width: 1024
     height: 576
     aspect: 1.778

From 0edb0821ba06a2c964647ee91f3b9b6731b01fcc Mon Sep 17 00:00:00 2001
From: jb-alvarado <jb@pixelcrusher.de>
Date: Sun, 28 Jun 2020 22:09:40 +0200
Subject: [PATCH 04/17] provide stream/hls output params

---
 ffplayout.yml              | 12 ++++++++++++
 ffplayout/output/hls.py    |  2 +-
 ffplayout/output/stream.py |  3 ++-
 ffplayout/utils.py         |  2 ++
 4 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/ffplayout.yml b/ffplayout.yml
index 6563dcc9..f23f4a33 100644
--- a/ffplayout.yml
+++ b/ffplayout.yml
@@ -98,6 +98,9 @@ out:
     helptext: The final playout compression. Set the settings to your needs.
         'mode' has the standard options 'desktop', 'hls', 'stream'. Self made outputs
         can be define, by adding script in output folder with an 'output' function inside.
+        'stream_output' is for streaming output, two ffmpeg instances are fired up, for
+        pre- and post-processing. 'hls_output' is for direct output to hls playlist,
+        without pre- and post-processing, mode must be 'hls'.
     mode: 'stream'
     service_name: "Live Stream"
     service_provider: "example.org"
@@ -113,5 +116,14 @@ out:
         -c:a aac
         -ar 44100
         -b:a 128k
+    stream_output: >-
         -flags +global_header
         -f flv "rtmp://localhost/live/stream"
+    hls_output: >-
+        -flags +cgop
+        -f hls
+        -hls_time 6
+        -hls_list_size 600
+        -hls_delete_threshold 30
+        -hls_flags append_list+delete_segments+omit_endlist+program_date_time
+        /var/www/srs/live/stream.m3u8
diff --git a/ffplayout/output/hls.py b/ffplayout/output/hls.py
index 7efe4eba..6a821df8 100644
--- a/ffplayout/output/hls.py
+++ b/ffplayout/output/hls.py
@@ -40,7 +40,7 @@ def output():
                         '-metadata', 'service_name=' + _playout.name,
                         '-metadata', 'service_provider=' + _playout.provider,
                         '-metadata', 'year={}'.format(year)
-                    ] + _playout.ffmpeg_param
+                    ] + _playout.ffmpeg_param + _playout.hls_output
 
                 _ff.encoder = Popen(cmd, stdin=PIPE, stderr=PIPE)
 
diff --git a/ffplayout/output/stream.py b/ffplayout/output/stream.py
index 200aa1dc..551bcb03 100644
--- a/ffplayout/output/stream.py
+++ b/ffplayout/output/stream.py
@@ -47,7 +47,8 @@ def output():
                 '-metadata', 'service_name=' + _playout.name,
                 '-metadata', 'service_provider=' + _playout.provider,
                 '-metadata', 'year={}'.format(year)
-            ] + _playout.ffmpeg_param, stdin=PIPE, stderr=PIPE)
+            ] + _playout.ffmpeg_param + _playout.stream_output,
+            stdin=PIPE, stderr=PIPE)
 
         enc_err_thread = Thread(target=ffmpeg_stderr_reader,
                                 args=(_ff.encoder.stderr, False))
diff --git a/ffplayout/utils.py b/ffplayout/utils.py
index 143bedcc..ae8e7cba 100644
--- a/ffplayout/utils.py
+++ b/ffplayout/utils.py
@@ -222,6 +222,8 @@ def load_config():
         _playout.name = cfg['out']['service_name']
         _playout.provider = cfg['out']['service_provider']
         _playout.ffmpeg_param = cfg['out']['ffmpeg_param'].split(' ')
+        _playout.stream_output = cfg['out']['stream_output'].split(' ')
+        _playout.hls_output = cfg['out']['hls_output'].split(' ')
 
         _init.load = False
 

From c5ed95e6810a15d3153969f6ef53de9b90584c56 Mon Sep 17 00:00:00 2001
From: jb-alvarado <jb@pixelcrusher.de>
Date: Sun, 28 Jun 2020 22:16:14 +0200
Subject: [PATCH 05/17] wrong index

---
 ffplayout.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ffplayout.yml b/ffplayout.yml
index f23f4a33..225abfa4 100644
--- a/ffplayout.yml
+++ b/ffplayout.yml
@@ -42,7 +42,7 @@ pre_process:
         loudness normalization. 'loud_*' can adjust the loudnorm filter. 'output_count'
         sets the outputs for the filtering, > 1 gives the option to use the same filters
         for multiple outputs. This outputs can be taken in 'ffmpeg_param', names will be
-        vout1, vout2; aout1, aout2 etc.
+        vout2, vout3; aout2, aout3 etc.
     width: 1024
     height: 576
     aspect: 1.778

From dc61a323cd3fae6b610c73cb0d4b48e4c3b93330 Mon Sep 17 00:00:00 2001
From: jb-alvarado <jb@pixelcrusher.de>
Date: Tue, 30 Jun 2020 20:46:12 +0200
Subject: [PATCH 06/17] remove thread_queue_size

---
 ffplayout/output/hls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ffplayout/output/hls.py b/ffplayout/output/hls.py
index 6a821df8..f955cfc4 100644
--- a/ffplayout/output/hls.py
+++ b/ffplayout/output/hls.py
@@ -35,7 +35,7 @@ def output():
                 messenger.info('Play: "{}"'.format(current_file))
                 cmd = [
                     'ffmpeg', '-v', _log.ff_level.lower(), '-hide_banner',
-                    '-nostats', '-re', '-thread_queue_size', '256'
+                    '-nostats', '-re'
                     ] + src_cmd + [
                         '-metadata', 'service_name=' + _playout.name,
                         '-metadata', 'service_provider=' + _playout.provider,

From 2a99cc08cbfa801764e826ddb58d1f1fe060219c Mon Sep 17 00:00:00 2001
From: jb-alvarado <jb@pixelcrusher.de>
Date: Wed, 1 Jul 2020 22:22:30 +0200
Subject: [PATCH 07/17] testing realtime filter for time correcting, #56

---
 ffplayout/filters.py    | 31 ++++++++++++++++++++++++-------
 ffplayout/output/hls.py |  2 +-
 ffplayout/utils.py      |  6 ++++++
 3 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/ffplayout/filters.py b/ffplayout/filters.py
index 1d057f90..b8475020 100644
--- a/ffplayout/filters.py
+++ b/ffplayout/filters.py
@@ -21,7 +21,7 @@ import math
 import os
 import re
 
-from .utils import _pre, _text
+from .utils import _global, _pre, _text
 
 # ------------------------------------------------------------------------------
 # building filters,
@@ -208,9 +208,23 @@ def extend_video(probe, duration, target_duration):
     return pad_filter
 
 
+def realtime_filter(duration, track=''):
+    speed_filter = ',{}realtime=speed=1'.format(track)
+
+    if _global.time_delta < 0:
+        speed = duration / (duration + _global.time_delta)
+
+        if speed < 1.1:
+            speed_filter = ',{}realtime=speed={}'.format(
+                track, speed
+            )
+
+    return speed_filter
+
 def split_filter(filter_type):
     map_node = []
     filter_prefix = ''
+    _filter = ''
 
     if filter_type == 'a':
         filter_prefix = 'a'
@@ -219,13 +233,13 @@ def split_filter(filter_type):
         for num in range(_pre.output_count):
             map_node.append('[{}out{}]'.format(filter_type, num + 1))
 
-        filter = ',{}split={}{}'.format(filter_prefix, _pre.output_count,
+        _filter = ',{}split={}{}'.format(filter_prefix, _pre.output_count,
                                         ''.join(map_node))
 
     else:
-        filter = '[{}out1]'.format(filter_type)
+        _filter = '[{}out1]'.format(filter_type)
 
-    return filter
+    return _filter
 
 
 def build_filtergraph(duration, seek, out, ad, ad_last, ad_next, probe, msg):
@@ -261,16 +275,19 @@ def build_filtergraph(duration, seek, out, ad, ad_last, ad_next, probe, msg):
         video_filter = 'null[v]'
 
     logo_filter = overlay_filter(out - seek, ad, ad_last, ad_next)
+    v_speed = realtime_filter(out - seek)
     v_split = split_filter('v')
     video_map = ['-map', '[vout1]']
     video_filter = [
-        '-filter_complex', '[0:v]{};{}{}'.format(
-            video_filter, logo_filter, v_split)]
+        '-filter_complex', '[0:v]{};{}{}{}'.format(
+            video_filter, logo_filter, v_speed, v_split)]
 
+    a_speed = realtime_filter(out - seek, 'a')
     a_split = split_filter('a')
     audio_map = ['-map', '[aout1]']
     audio_filter = [
-        '-filter_complex', '{}{}'.format(','.join(audio_chain), a_split)]
+        '-filter_complex', '{}{}{}'.format(','.join(audio_chain),
+                                           a_speed, a_split)]
 
     if probe.video[0]:
         return video_filter + audio_filter + video_map + audio_map
diff --git a/ffplayout/output/hls.py b/ffplayout/output/hls.py
index f955cfc4..4e8a272d 100644
--- a/ffplayout/output/hls.py
+++ b/ffplayout/output/hls.py
@@ -35,7 +35,7 @@ def output():
                 messenger.info('Play: "{}"'.format(current_file))
                 cmd = [
                     'ffmpeg', '-v', _log.ff_level.lower(), '-hide_banner',
-                    '-nostats', '-re'
+                    '-nostats'
                     ] + src_cmd + [
                         '-metadata', 'service_name=' + _playout.name,
                         '-metadata', 'service_provider=' + _playout.provider,
diff --git a/ffplayout/utils.py b/ffplayout/utils.py
index ae8e7cba..4b85d2e7 100644
--- a/ffplayout/utils.py
+++ b/ffplayout/utils.py
@@ -121,6 +121,7 @@ _playout = SimpleNamespace()
 
 _init = SimpleNamespace(load=True)
 _ff = SimpleNamespace(decoder=None, encoder=None)
+_global = SimpleNamespace(time_delta=0)
 
 
 def str_to_sec(s):
@@ -675,6 +676,11 @@ def check_sync(delta):
     """
     check that we are in tolerance time
     """
+
+    if _playlist.mode and _playlist.start and _playlist.length:
+        # save time delta to global variable for syncing
+        _global.time_delta = delta
+
     if _general.stop and abs(delta) > _general.threshold:
         messenger.error(
             'Sync tolerance value exceeded with {0:.2f} seconds,\n'

From 454a020bc52e861eb1997e7f6d60afb65a68c5f4 Mon Sep 17 00:00:00 2001
From: Jonathan Baecker <jb@pixelcrusher.de>
Date: Thu, 2 Jul 2020 13:42:43 +0200
Subject: [PATCH 08/17] fix help text

---
 ffplayout.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/ffplayout.yml b/ffplayout.yml
index 225abfa4..12423aab 100644
--- a/ffplayout.yml
+++ b/ffplayout.yml
@@ -73,9 +73,8 @@ playlist:
 
 storage:
     helptext: Play ordered or ramdomly files from path. 'filler_clip' is for fill
-        the end to reach 24 hours, it will loop when is necessary extensions search
-        only files with this extension, can be a list. Set 'shuffle' to 'True' to
-        pick files randomly.
+        the end to reach 24 hours, it will loop when is necessary. 'extensions' search
+        only files with this extension. Set 'shuffle' to 'True' to pick files randomly.
     path: "/mediaStorage"
     filler_clip: "/mediaStorage/filler/filler.mp4"
     extensions:

From 3921f0fccc8c009ce54d1a37647ef65d79141657 Mon Sep 17 00:00:00 2001
From: jb-alvarado <jb@pixelcrusher.de>
Date: Thu, 2 Jul 2020 20:54:06 +0200
Subject: [PATCH 09/17] rename key and add use_realtime

---
 ffplayout.yml      | 28 +++++++++++++++-------------
 ffplayout/utils.py | 33 +++++++++++++++++----------------
 2 files changed, 32 insertions(+), 29 deletions(-)

diff --git a/ffplayout.yml b/ffplayout.yml
index 12423aab..eee6b81e 100644
--- a/ffplayout.yml
+++ b/ffplayout.yml
@@ -31,18 +31,19 @@ logging:
     log_level: "DEBUG"
     ffmpeg_level: "ERROR"
 
-pre_process:
-    helptext: Settings for the pre_process. All clips get prepared in that way,
-        so the input for the final compression is unique. 'aspect' must be a float
-        number. 'logo' is only used if the path exist. 'logo_scale' scale the logo to
-        target size, leave it blank when no scaling is needed, format is 'number:number',
-        for example '100:-1' for proportional scaling. With 'logo_opacity' logo can
-        become transparent. With 'logo_filter' 'overlay=W-w-12:12' you can modify
-        the logo position. With 'use_loudnorm' you can activate single pass EBU R128
-        loudness normalization. 'loud_*' can adjust the loudnorm filter. 'output_count'
-        sets the outputs for the filtering, > 1 gives the option to use the same filters
-        for multiple outputs. This outputs can be taken in 'ffmpeg_param', names will be
-        vout2, vout3; aout2, aout3 etc.
+processing:
+    helptext: Default processing, for all clips that they get prepared in that way,
+        so the output is unique. 'aspect' must be a float number. 'logo' is only used
+        if the path exist. 'logo_scale' scale the logo to target size, leave it blank
+        when no scaling is needed, format is 'number:number', for example '100:-1'
+        for proportional scaling. With 'logo_opacity' logo can become transparent.
+        With 'logo_filter' 'overlay=W-w-12:12' you can modify the logo position.
+        With 'use_loudnorm' you can activate single pass EBU R128 loudness normalization.
+        'loud_*' can adjust the loudnorm filter. 'output_count' sets the outputs for
+        the filtering, > 1 gives the option to use the same filters for multiple outputs.
+        This outputs can be taken in 'ffmpeg_param', names will be vout2, vout3;
+        aout2, aout2 etc.'use_realtime' is realtime filter, it works not in all scenarios,
+        but for example is necessary for hls output.
     width: 1024
     height: 576
     aspect: 1.778
@@ -57,9 +58,10 @@ pre_process:
     loud_TP: -1.5
     loud_LRA: 11
     output_count: 1
+    use_realtime: false
 
 playlist:
-    helptext: Set 'playlist_mode' to 'False' if you want to play clips from the [STORAGE]
+    helptext: Set 'playlist_mode' to 'False' if you want to play clips from the 'storage'
         section. Put only the root path here, for example '/playlists' subfolders
         are readed by the script. Subfolders needs this structur '/playlists/2018/01'
         (/playlists/year/month). 'day_start' means at which time the playlist should
diff --git a/ffplayout/utils.py b/ffplayout/utils.py
index 4b85d2e7..5064cbec 100644
--- a/ffplayout/utils.py
+++ b/ffplayout/utils.py
@@ -179,16 +179,16 @@ def load_config():
     _mail.recip = cfg['mail']['recipient']
     _mail.level = cfg['mail']['mail_level']
 
-    _pre.add_logo = cfg['pre_process']['add_logo']
-    _pre.logo = cfg['pre_process']['logo']
-    _pre.logo_scale = cfg['pre_process']['logo_scale']
-    _pre.logo_filter = cfg['pre_process']['logo_filter']
-    _pre.logo_opacity = cfg['pre_process']['logo_opacity']
-    _pre.add_loudnorm = cfg['pre_process']['add_loudnorm']
-    _pre.loud_i = cfg['pre_process']['loud_I']
-    _pre.loud_tp = cfg['pre_process']['loud_TP']
-    _pre.loud_lra = cfg['pre_process']['loud_LRA']
-    _pre.output_count = cfg['pre_process']['output_count']
+    _pre.add_logo = cfg['processing']['add_logo']
+    _pre.logo = cfg['processing']['logo']
+    _pre.logo_scale = cfg['processing']['logo_scale']
+    _pre.logo_filter = cfg['processing']['logo_filter']
+    _pre.logo_opacity = cfg['processing']['logo_opacity']
+    _pre.add_loudnorm = cfg['processing']['add_loudnorm']
+    _pre.loud_i = cfg['processing']['loud_I']
+    _pre.loud_tp = cfg['processing']['loud_TP']
+    _pre.loud_lra = cfg['processing']['loud_LRA']
+    _pre.output_count = cfg['processing']['output_count']
 
     _playlist.mode = cfg['playlist']['playlist_mode']
     _playlist.path = cfg['playlist']['path']
@@ -212,12 +212,13 @@ def load_config():
         _log.level = cfg['logging']['log_level']
         _log.ff_level = cfg['logging']['ffmpeg_level']
 
-        _pre.w = cfg['pre_process']['width']
-        _pre.h = cfg['pre_process']['height']
-        _pre.aspect = cfg['pre_process']['aspect']
-        _pre.fps = cfg['pre_process']['fps']
-        _pre.v_bitrate = cfg['pre_process']['width'] * 50
-        _pre.v_bufsize = cfg['pre_process']['width'] * 50 / 2
+        _pre.w = cfg['processing']['width']
+        _pre.h = cfg['processing']['height']
+        _pre.aspect = cfg['processing']['aspect']
+        _pre.fps = cfg['processing']['fps']
+        _pre.v_bitrate = cfg['processing']['width'] * 50
+        _pre.v_bufsize = cfg['processing']['width'] * 50 / 2
+        _pre.realtime = cfg['processing']['use_realtime']
 
         _playout.mode = cfg['out']['mode']
         _playout.name = cfg['out']['service_name']

From fb165e08fde902e4254aeb506273b2ecf8569783 Mon Sep 17 00:00:00 2001
From: jb-alvarado <jb@pixelcrusher.de>
Date: Thu, 2 Jul 2020 20:54:45 +0200
Subject: [PATCH 10/17] only use realtime filter when option is true

---
 ffplayout/filters.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/ffplayout/filters.py b/ffplayout/filters.py
index b8475020..f1e7d130 100644
--- a/ffplayout/filters.py
+++ b/ffplayout/filters.py
@@ -209,15 +209,16 @@ def extend_video(probe, duration, target_duration):
 
 
 def realtime_filter(duration, track=''):
-    speed_filter = ',{}realtime=speed=1'.format(track)
+    speed_filter = ''
 
-    if _global.time_delta < 0:
-        speed = duration / (duration + _global.time_delta)
+    if _pre.realtime:
+        speed_filter = ',{}realtime=speed=1'.format(track)
 
-        if speed < 1.1:
-            speed_filter = ',{}realtime=speed={}'.format(
-                track, speed
-            )
+        if _global.time_delta < 0:
+            speed = duration / (duration + _global.time_delta)
+
+            if speed < 1.1:
+                speed_filter = ',{}realtime=speed={}'.format(track, speed)
 
     return speed_filter
 

From ae2aa44901b58bb964e1db198e01276d892464fc Mon Sep 17 00:00:00 2001
From: jb-alvarado <jb@pixelcrusher.de>
Date: Thu, 2 Jul 2020 20:55:42 +0200
Subject: [PATCH 11/17] rename enc thread, add ts cleanup thread

---
 ffplayout/output/hls.py | 36 +++++++++++++++++++++++++++++++-----
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/ffplayout/output/hls.py b/ffplayout/output/hls.py
index 4e8a272d..765397cf 100644
--- a/ffplayout/output/hls.py
+++ b/ffplayout/output/hls.py
@@ -1,3 +1,6 @@
+import os
+import re
+from glob import iglob
 from subprocess import PIPE, Popen
 from threading import Thread
 
@@ -8,6 +11,25 @@ from ffplayout.utils import (_ff, _log, _playlist, _playout,
                              stdin_args, terminate_processes)
 
 
+def clean_ts():
+    playlists = re.findall(r'[/\w.]+m3u8', _playout.hls_output)
+
+    for playlist in playlists:
+        test_num = 0
+        hls_path = os.path.dirname(playlist)
+        with open(playlist, 'r') as m3u8:
+            for line in m3u8:
+                if '.ts' in line:
+                    test_num = int(re.findall(r'(\d+).ts', line)[0])
+                    break
+
+        for ts_file in iglob(os.path.join(hls_path, '*.ts')):
+            ts_num = int(re.findall(r'(\d+).ts', ts_file)[0])
+
+            if test_num > ts_num:
+                os.remove(ts_file)
+
+
 def output():
     """
     this output is hls output, no preprocess is needed.
@@ -44,11 +66,15 @@ def output():
 
                 _ff.encoder = Popen(cmd, stdin=PIPE, stderr=PIPE)
 
-                enc_thread = Thread(target=ffmpeg_stderr_reader,
-                                    args=(_ff.encoder.stderr, True))
-                enc_thread.daemon = True
-                enc_thread.start()
-                enc_thread.join()
+                stderr_reader_thread = Thread(target=ffmpeg_stderr_reader,
+                                              args=(_ff.encoder.stderr, False))
+                stderr_reader_thread.daemon = True
+                stderr_reader_thread.start()
+                stderr_reader_thread.join()
+
+                ts_cleaning_thread = Thread(target=clean_ts)
+                ts_cleaning_thread.daemon = True
+                ts_cleaning_thread.start()
 
         except BrokenPipeError:
             messenger.error('Broken Pipe!')

From 957b7c5ac72dde1921ee6c64d6bbe811489b0944 Mon Sep 17 00:00:00 2001
From: jb-alvarado <jb@pixelcrusher.de>
Date: Thu, 2 Jul 2020 22:20:15 +0200
Subject: [PATCH 12/17] we have to check a list, not a string

---
 ffplayout/output/hls.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ffplayout/output/hls.py b/ffplayout/output/hls.py
index 765397cf..50638658 100644
--- a/ffplayout/output/hls.py
+++ b/ffplayout/output/hls.py
@@ -12,7 +12,7 @@ from ffplayout.utils import (_ff, _log, _playlist, _playout,
 
 
 def clean_ts():
-    playlists = re.findall(r'[/\w.]+m3u8', _playout.hls_output)
+    playlists = [p for p in _playout.hls_output if 'm3u8' in p]
 
     for playlist in playlists:
         test_num = 0

From 0f31bbf1bdc3d7274a937c18e0fe6867d788e566 Mon Sep 17 00:00:00 2001
From: jb-alvarado <jb@pixelcrusher.de>
Date: Thu, 2 Jul 2020 22:22:05 +0200
Subject: [PATCH 13/17] add debug info

---
 ffplayout/output/hls.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ffplayout/output/hls.py b/ffplayout/output/hls.py
index 50638658..aead2c3a 100644
--- a/ffplayout/output/hls.py
+++ b/ffplayout/output/hls.py
@@ -15,6 +15,7 @@ def clean_ts():
     playlists = [p for p in _playout.hls_output if 'm3u8' in p]
 
     for playlist in playlists:
+        messenger.debug('cleanup *.ts files from: "{}"'.format(playlist))
         test_num = 0
         hls_path = os.path.dirname(playlist)
         with open(playlist, 'r') as m3u8:

From 56bc176b2d963f34cb1d7e2fe5d771557bab2860 Mon Sep 17 00:00:00 2001
From: jb-alvarado <jb@pixelcrusher.de>
Date: Thu, 2 Jul 2020 22:30:06 +0200
Subject: [PATCH 14/17] add doc string

---
 ffplayout/output/hls.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ffplayout/output/hls.py b/ffplayout/output/hls.py
index aead2c3a..25ec4b04 100644
--- a/ffplayout/output/hls.py
+++ b/ffplayout/output/hls.py
@@ -12,6 +12,12 @@ from ffplayout.utils import (_ff, _log, _playlist, _playout,
 
 
 def clean_ts():
+    """
+    this function get all *.m3u8 playlists from config,
+    read lines from them until it founds first *.ts file,
+    then it checks if files on harddrive are older then this first *.ts
+    and if so delete them
+    """
     playlists = [p for p in _playout.hls_output if 'm3u8' in p]
 
     for playlist in playlists:

From c8f841c109219846470883a4f0a741be7059af04 Mon Sep 17 00:00:00 2001
From: jb-alvarado <jb@pixelcrusher.de>
Date: Thu, 9 Jul 2020 22:33:30 +0200
Subject: [PATCH 15/17] add infos about new outputs

---
 README.md | 40 +++++++++++++++++++++++++++-------------
 1 file changed, 27 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index f8e7a4a5..cb1ac067 100644
--- a/README.md
+++ b/README.md
@@ -1,13 +1,14 @@
 **ffplayout-engine**
 ================
+
 [![made-with-python](https://img.shields.io/badge/Made%20with-Python-1f425f.svg)](https://www.python.org/)
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 
 The purpose with ffplayout is to provide a 24/7 broadcasting solution that plays a *json* playlist for every day, while keeping the current playlist editable.
 
-#### Check [ffplayout-gui](https://github.com/ffplayout/ffplayout-gui): web-based GUI for ffplayout.
+**Check [ffplayout-gui](https://github.com/ffplayout/ffplayout-gui): web-based GUI for ffplayout**
 
-Features
+**Features**
 -----
 
 - have all values in a separate config file
@@ -29,17 +30,22 @@ Features
 - on posix systems ffplayout can reload config with *SIGHUP*
 - logging to files, or colored output to console
 - add filters to input, if is necessary to match output stream:
-    - **yadif** (deinterlacing)
-    - **pad** (letterbox or pillarbox to fit aspect)
-    - **fps** (change fps)
-    - **scale** (fit target resolution)
-    - **aevalsrc** (if video have no audio)
-    - **apad** (add silence if audio duration is to short)
-    - **tpad** (add black frames if video duration is to short)
-
+  - **yadif** (deinterlacing)
+  - **pad** (letterbox or pillarbox to fit aspect)
+  - **fps** (change fps)
+  - **scale** (fit target resolution)
+  - **aevalsrc** (if video have no audio)
+  - **apad** (add silence if audio duration is to short)
+  - **tpad** (add black frames if video duration is to short)
+- different types of [output](https://github.com/ffplayout/ffplayout-engine/wiki/Outputs):
+  - **stream**
+  - **desktop**
+  - **HLS**
+  - **custom**
 
 Requirements
 -----
+
 - python version 3.6+
 - python module **watchdog** (only when `playlist_mode: False`)
 - python module **colorama** if you are on windows
@@ -81,11 +87,14 @@ JSON Playlist Example
 }
 ```
 
-#### Warning:
+**Warning**
+-----
+
 (Endless) streaming over multiple days will only work when config have **day_start** value and the **length** value is **24 hours**. If you need only some hours for every day, use a *cron* job, or something similar.
 
 Remote source from URL
 -----
+
 You can use sources from remote URL in that way:
 
 ```json
@@ -97,17 +106,21 @@ You can use sources from remote URL in that way:
             "source": "https://example.org/big_buck_bunny.webm"
         }
 ```
+
 But be careful with it, better test it multiple times!
 
 More informations in [Wiki](https://github.com/ffplayout/ffplayout-engine/wiki/Remote-URL-Source)
 
 Installation
 -----
+
 Check [INSTALL.md](docs/INSTALL.md)
 
 Start with Arguments
 -----
+
 ffplayout also allows the passing of parameters:
+
 - `-c, --config` use given config file
 - `-d, --desktop` preview on desktop
 - `-f, --folder` use folder for playing
@@ -119,10 +132,11 @@ ffplayout also allows the passing of parameters:
 
 You can run the command like:
 
-```
+```SHELL
 ./ffplayout.py -l none -p ~/playlist.json -d -s now -t none
 ```
 
 Play on Desktop
 -----
-For playing on desktop use `-d` argument or set `preview: True` in config under `out:`.
+
+For playing on desktop use `-d` argument or set `mode: 'desktop'` in config under `out:`.

From 002edae315ad02534c910bb0a066918d10b28a5a Mon Sep 17 00:00:00 2001
From: jb-alvarado <jb@pixelcrusher.de>
Date: Thu, 9 Jul 2020 22:39:35 +0200
Subject: [PATCH 16/17] updates

---
 requirements.txt | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 2448a3b8..26dda6cf 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
 colorama==0.4.3
 pathtools==0.1.2
-PyYAML==5.3
-watchdog==0.10.1
+PyYAML==5.3.1
+watchdog==0.10.3

From ddfd0c2fbc047ee0189f2a7b7cd5398eb314c1c9 Mon Sep 17 00:00:00 2001
From: jb-alvarado <jb@pixelcrusher.de>
Date: Thu, 9 Jul 2020 22:44:14 +0200
Subject: [PATCH 17/17] add blank line and intend

---
 ffplayout/filters.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ffplayout/filters.py b/ffplayout/filters.py
index f1e7d130..d854fa84 100644
--- a/ffplayout/filters.py
+++ b/ffplayout/filters.py
@@ -222,6 +222,7 @@ def realtime_filter(duration, track=''):
 
     return speed_filter
 
+
 def split_filter(filter_type):
     map_node = []
     filter_prefix = ''
@@ -235,7 +236,7 @@ def split_filter(filter_type):
             map_node.append('[{}out{}]'.format(filter_type, num + 1))
 
         _filter = ',{}split={}{}'.format(filter_prefix, _pre.output_count,
-                                        ''.join(map_node))
+                                         ''.join(map_node))
 
     else:
         _filter = '[{}out1]'.format(filter_type)