From a3f8728d9de3ec36282c62e642df8c496d2d4e8d Mon Sep 17 00:00:00 2001
From: Christopher Vagnetoft <cvagnetoft@gmail.com>
Date: Sat, 11 Feb 2017 23:37:35 +0100
Subject: [PATCH] Added autoloader to plugin loader, executor improvements

---
 plugins/executor/Helper.php              |  38 +++++
 plugins/executor/Operation.php           |  87 +++++++++++
 plugins/executor/Parser.php              | 105 +++++++++++++
 plugins/executor/README.md               |  31 ++++
 plugins/executor/Script.php              |  84 +++++++++++
 plugins/executor/plugin.php              | 184 +----------------------
 plugins/transcode/plugin.php             |   1 -
 presets/executor/executor-scenesplit.yml | 111 ++++++++++++++
 src/Application.php                      |  10 ++
 src/Plugin.php                           |   4 +-
 10 files changed, 474 insertions(+), 181 deletions(-)
 create mode 100644 plugins/executor/Helper.php
 create mode 100644 plugins/executor/Operation.php
 create mode 100644 plugins/executor/Parser.php
 create mode 100644 plugins/executor/Script.php
 create mode 100644 presets/executor/executor-scenesplit.yml

diff --git a/plugins/executor/Helper.php b/plugins/executor/Helper.php
new file mode 100644
index 0000000..2d6567c
--- /dev/null
+++ b/plugins/executor/Helper.php
@@ -0,0 +1,38 @@
+<?php
+
+namespace VfxApply\Plugin\Executor;
+
+use VfxApply\Plugin;
+use VfxApply\Input;
+use VfxApply\Output;
+use VfxApply\Preset;
+
+class Helper
+{
+    protected $body;
+
+    protected $script;
+
+    public function __construct($body)
+    {
+        $this->body = $body;
+    }
+
+    public function setScript(Script $script)
+    {
+        $this->script = $script;
+    }
+
+    public function call(array $env)
+    {
+        $body = "<?php ".preg_replace_callback('/(\{\{\%([a-z]+)\}\})/', function ($m) use ($env) {
+            return $env[$m[2]];
+        }, $this->body);
+
+        $tmpname = tempnam(null, "vfxhelper");
+        file_put_contents($tmpname, $body);
+        passthru("php {$tmpname}");
+        unlink($tmpname);
+    }
+}
+
diff --git a/plugins/executor/Operation.php b/plugins/executor/Operation.php
new file mode 100644
index 0000000..93611a8
--- /dev/null
+++ b/plugins/executor/Operation.php
@@ -0,0 +1,87 @@
+<?php
+
+namespace VfxApply\Plugin\Executor;
+
+use VfxApply\Plugin;
+use VfxApply\Input;
+use VfxApply\Output;
+use VfxApply\Preset;
+
+class Operation
+{
+    protected $name;
+
+    protected $info;
+
+    protected $parser;
+
+    protected $command;
+
+    protected $helper;
+
+    protected $script;
+
+    public function __construct($name, array $data)
+    {
+        $_ = function($a,$k,$d=null) { return empty($a[$k])?$d:$a[$k]; };
+        $this->name = $name;
+        $this->info = $_($data,'info');
+        $this->command = $_($data,'exec');
+        $this->helper = $_($data,'call');
+        $this->parser = new Parser($_($data,'parse'), $_($data,'extract'));
+    }
+
+    public function getName()
+    {
+        return $this->name;
+    }
+
+    public function getInfo()
+    {
+        return $this->info;
+    }
+
+    public function setScript(Script $script)
+    {
+        $this->script = $script;
+    }
+
+    public function execute(array $env, $dialog)
+    {
+        if ($this->helper) {
+            $helper = $this->script->getHelper($this->helper);
+            $helper->call($env);
+        }
+
+        if (!$this->command) {
+            return;
+        }
+        $cmdl = $this->script->parseVariable($this->command, $env);
+        //printf(" cmd: %s\n", $this->command);
+        //printf("   eval: %s\n", $cmdl);
+
+        $this->parser->prepareExtractors($this->script, $env);
+
+        $descr = [
+            0 => [ 'pipe', 'r' ],
+            1 => [ 'pipe', 'w' ],
+            2 => [ 'pipe', 'w' ]
+        ];
+        echo "Exec: ".$cmdl."\n";
+        $proc = proc_open($cmdl, $descr, $pipes);
+        $tot = (int)$env['frames']-1;
+        printf("\r%s [%s]", $this->info, $this->name);
+        $this->parser->parse($pipes, function ($status) use ($tot, $dialog) {
+            $curr = (int)$status['frame'];
+            $pc = min(100,100/$tot*$curr);
+            $dialog->setProgress($pc);
+            $out = sprintf("%s [%s]: %.1f%%", $this->info, $this->name, $pc);
+            $dialog->setText($out);
+            echo "\r{$out}";
+        });
+        proc_close($proc);
+        printf("\n");
+
+    }
+}
+
diff --git a/plugins/executor/Parser.php b/plugins/executor/Parser.php
new file mode 100644
index 0000000..c182881
--- /dev/null
+++ b/plugins/executor/Parser.php
@@ -0,0 +1,105 @@
+<?php
+
+namespace VfxApply\Plugin\Executor;
+
+use VfxApply\Plugin;
+use VfxApply\Input;
+use VfxApply\Output;
+use VfxApply\Preset;
+
+class Parser
+{
+    const STDOUT = 1;
+    const STDERR = 2;
+
+    /** @var array Resources */
+    protected $streams = [];
+    /** @var string The expression to match (preg match) */
+    protected $expression;
+    /** @var int The stream to watch */
+    protected $source;
+    /** @var array Index to name mappings */
+    protected $names = [];
+
+    protected $extractors = [];
+
+    public function __construct(array $parser=null, $extractors=null)
+    {
+        $_ = function($a,$k,$d=null) { return empty($a[$k])?$d:$a[$k]; };
+        $this->expression = $_($parser,'regex');
+        $this->source = $_($parser,'from');
+        foreach ((array)$parser as $k=>$v) {
+            if (is_numeric($v)) {
+                $this->names[$k] = $v;
+            }
+        }
+        if (is_array($extractors))
+            $this->extractors = $extractors;
+    }
+
+    public function prepareExtractors(Script $script, array $env)
+    {
+        foreach ($this->extractors as $i=>$extractor) {
+            if (array_key_exists('write',$extractor)) {
+                $write = $this->extractors[$i]['write'];
+                $write = $script->parseVariable($write, $env);
+                $this->extractors[$i]['write'] = trim($write,"'");
+            }
+        }
+    }
+
+    public function parse(array $streams, callable $callback)
+    {
+        $this->streams = $streams;
+        $match = [ 'stdout' => 1, 'stderr' => 2 ];
+        $source = $this->source?:'stdout';
+
+        foreach ($this->streams as $stream)
+            stream_set_blocking($stream, false);
+
+        while (!feof($this->streams[1])) {
+            $stdout = fread($this->streams[1],8192);
+            $stderr = fread($this->streams[2],8192);
+            $this->parseExtractors($stdout, $stderr);
+            //($stdout) && printf("OUT: <<%s>>\n", $stdout);
+            //($stderr) && printf("ERR: <<%s>>\n", $stderr);
+            if (($stdout && $this->expression) && ($source=='stdout'))
+                if (preg_match($this->expression, $stdout, $match))
+                    call_user_func($callback, $this->parseNames($match));
+            if (($stderr && $this->expression) && ($source=='stderr'))
+                if (preg_match($this->expression, $stderr, $match))
+                    call_user_func($callback, $this->parseNames($match));
+            usleep(10000);
+        }
+    }
+
+    protected function parseExtractors($stdout, $stderr)
+    {
+        foreach ($this->extractors as $extractor) {
+            switch ($extractor['from']) {
+                case 'stdout': $buf = $stdout; break;
+                case 'stderr': $buf = $stderr; break;
+            }
+            if ($buf == "") continue;
+            $lines = explode("\n", $buf);
+            foreach ($lines as $line) {
+                if (preg_match($extractor['regex'], $line)) {
+                    $fo = fopen($extractor['write'], "a+");
+                    fwrite($fo, trim($line)."\n");
+                    fclose($fo);
+                }
+            }
+        }
+    }
+
+    protected function parseNames(array $match)
+    {
+        $ret = [];
+        foreach ($this->names as $k=>$index) {
+            $ret[$k] = $match[$index];
+        }
+        return $ret;
+    }
+
+}
+
diff --git a/plugins/executor/README.md b/plugins/executor/README.md
index 4ac42fa..5d8efc3 100644
--- a/plugins/executor/README.md
+++ b/plugins/executor/README.md
@@ -64,3 +64,34 @@ To parse the output, add a `parse` key to the command block.
             from:     <- stream for comparison (stdout or stderr)
             frame:    <- index of frame number, or leave out
             fps:      <- index of frames per second, or leave out
+
+### Extracting output
+
+While you can only have a single parser, you can have multiple extractors.
+An extractor matches a pattern on a line, and then writes it to the destination
+file.
+
+        extract:
+            # Write everything matching '[Parsed_blackframe' to a file
+            - { from: 'stderr', regex: '/^\[Parsed_blackframe/', write:'{%scenes}' }
+
+
+## Helpers
+
+Helpers can be defined in the preset. These are essentially embedded scripts
+written in PHP that have access to the environment of the preset.
+
+Helpers are not executed, but rather called:
+
+        ...
+            split:
+                info: Splitting video
+                call: scenesplitter
+            cleanup:
+                exec: "rm {%scenes}"
+        helpers:
+            scenesplitter: |
+                /*
+                 * This is the PHP script
+                 *
+                ...
diff --git a/plugins/executor/Script.php b/plugins/executor/Script.php
new file mode 100644
index 0000000..8f7d148
--- /dev/null
+++ b/plugins/executor/Script.php
@@ -0,0 +1,84 @@
+<?php
+
+namespace VfxApply\Plugin\Executor;
+
+use VfxApply\Plugin;
+use VfxApply\Input;
+use VfxApply\Output;
+use VfxApply\Preset;
+
+class Script
+{
+    /** @var array Variables */
+    protected $vars = [];
+
+    protected $plugin;
+
+    public function setPlugin($plugin)
+    {
+        $this->plugin = $plugin;
+    }
+
+    public function set($k,$v)
+    {
+        $this->vars[$k] = $v;
+    }
+
+    public function addOperation(Operation $operation)
+    {
+        $operation->setScript($this);
+        $this->operations[] = $operation;
+    }
+
+    public function addHelper($name, Helper $helper)
+    {
+        $helper->setScript($this);
+        $this->helpers[$name] = $helper;
+    }
+
+    public function getHelper($name)
+    {
+        return $this->helpers[$name];
+    }
+
+    public function execute(array $env)
+    {
+        $env = $this->buildEnvironment($env, $this->vars);
+        //print_r($env);
+        foreach ($this->operations as $operation) {
+            //printf("Executing: %s\n", $operation->getInfo());
+            $dialog = $this->plugin->createDialog(DIALOG_PROGRESS, $operation->getInfo());
+            $dialog->show();
+            $operation->execute($env, $dialog);
+            unset($dialog);
+        }
+    }
+
+    private function buildEnvironment(array $env, array $vars)
+    {
+        $vars = array_merge($env, $vars);
+        foreach ($vars as $k=>$var) {
+            if (is_array($var)) {
+                $esc = empty($var['escape'])?false:$var['escape'];
+                $var = preg_replace_callback('/\{\%([a-z]+?)\}/i', function ($match) use (&$vars,$var) {
+                    $k = $match[1];
+                    if (empty($vars[$k])) return null;
+                    return $vars[$k];
+                }, $var['value']);
+                if ($esc) $var = escapeshellarg($var);
+                $vars[$k] = $var;
+            }
+        }
+        return $vars;
+    }
+
+    public function parseVariable($value, array $env)
+    {
+        return preg_replace_callback('/\{\%([a-z]+?)\}/i', function ($match) use (&$env) {
+            $k = $match[1];
+            if (empty($env[$k])) return null;
+            return $env[$k];
+        }, $value);
+    }
+}
+
diff --git a/plugins/executor/plugin.php b/plugins/executor/plugin.php
index f58df04..34ba702 100644
--- a/plugins/executor/plugin.php
+++ b/plugins/executor/plugin.php
@@ -27,6 +27,7 @@ class ExecutorPlugin extends Plugin
         $env['output'] = escapeshellarg($env['uoutput']);
         
         $script = $this->loadScript($preset);
+        $script->setPlugin($this);
         $script->execute($env);
     }
 
@@ -41,187 +42,14 @@ class ExecutorPlugin extends Plugin
             $op = new Operation($name, $step);
             $script->addOperation($op);
         }
+        foreach ((array)$preset->get('helpers') as $name=>$helper) {
+            $hs = new Helper($helper);
+            $script->addHelper($name, $hs);
+        }
         return $script;
     }
 
 }
 
-class Script
-{
-    /** @var array Variables */
-    protected $vars = [];
-
-    public function set($k,$v)
-    {
-        $this->vars[$k] = $v;
-    }
-
-    public function addOperation(Operation $operation)
-    {
-        $operation->setScript($this);
-        $this->operations[] = $operation;
-    }
-
-    public function execute(array $env)
-    {
-        $env = $this->buildEnvironment($env, $this->vars);
-        //print_r($env);
-        foreach ($this->operations as $operation) {
-            //printf("Executing: %s\n", $operation->getInfo());
-            $operation->execute($env);
-        }
-    }
-
-    private function buildEnvironment(array $env, array $vars)
-    {
-        $vars = array_merge($env, $vars);
-        foreach ($vars as $k=>$var) {
-            if (is_array($var)) {
-                $esc = empty($var['escape'])?false:$var['escape'];
-                $var = preg_replace_callback('/\{\%([a-z]+?)\}/i', function ($match) use (&$vars,$var) {
-                    $k = $match[1];
-                    if (empty($vars[$k])) return null;
-                    return $vars[$k];
-                }, $var['value']);
-                if ($esc) $var = escapeshellarg($var);
-                $vars[$k] = $var;
-            }
-        }
-        return $vars;
-    }
-
-    public function parseVariable($value, array $env)
-    {
-        return preg_replace_callback('/\{\%([a-z]+?)\}/i', function ($match) use (&$env) {
-            $k = $match[1];
-            if (empty($env[$k])) return null;
-            return $env[$k];
-        }, $value);
-    }
-}
-
-class Operation
-{
-    protected $name;
-
-    protected $info;
-
-    protected $parser;
-
-    protected $command;
-
-    protected $script;
-
-    public function __construct($name, array $data)
-    {
-        $_ = function($a,$k,$d=null) { return empty($a[$k])?$d:$a[$k]; };
-        $this->name = $name;
-        $this->info = $_($data,'info');
-        $this->command = $_($data,'exec');
-        $this->parser = new Parser($_($data,'parse'));
-    }
-
-    public function getName()
-    {
-        return $this->name;
-    }
-
-    public function getInfo()
-    {
-        return $this->info;
-    }
-
-    public function setScript(Script $script)
-    {
-        $this->script = $script;
-    }
-
-    public function execute(array $env)
-    {
-        $cmdl = $this->script->parseVariable($this->command, $env);
-        //printf(" cmd: %s\n", $this->command);
-        //printf("   eval: %s\n", $cmdl);
-
-        $descr = [
-            0 => [ 'pipe', 'r' ],
-            1 => [ 'pipe', 'w' ],
-            2 => [ 'pipe', 'w' ]
-        ];
-        echo "Exec: ".$cmdl."\n";
-        $proc = proc_open($cmdl, $descr, $pipes);
-        $tot = (int)$env['frames']-1;
-        printf("\r%s [%s]", $this->info, $this->name);
-        $this->parser->parse($pipes, function ($status) use ($tot) {
-            $curr = (int)$status['frame'];
-            $pc = min(100,100/$tot*$curr);
-            printf("\r%s [%s]: %.1f%%", $this->info, $this->name, $pc);
-        });
-        proc_close($proc);
-        printf("\n");
-
-    }
-}
-
-class Parser
-{
-    const STDOUT = 1;
-    const STDERR = 2;
-
-    /** @var array Resources */
-    protected $streams = [];
-    /** @var string The expression to match (preg match) */
-    protected $expression;
-    /** @var int The stream to watch */
-    protected $source;
-    /** @var array Index to name mappings */
-    protected $names = [];
-
-    public function __construct(array $parser=null)
-    {
-        $_ = function($a,$k,$d=null) { return empty($a[$k])?$d:$a[$k]; };
-        $this->expression = $_($parser,'regex');
-        $this->source = $_($parser,'from');
-        foreach ((array)$parser as $k=>$v) {
-            if (is_numeric($v)) {
-                $this->names[$k] = $v;
-            }
-        }
-    }
-
-    public function parse(array $streams, callable $callback)
-    {
-        $this->streams = $streams;
-        $match = [ 'stdout' => 1, 'stderr' => 2 ];
-        $source = $this->source?:'stdout';
-
-        foreach ($this->streams as $stream)
-            stream_set_blocking($stream, false);
-
-        while (!feof($this->streams[1])) {
-            $stdout = fread($this->streams[1],8192);
-            $stderr = fread($this->streams[2],8192);
-            //($stdout) && printf("OUT: <<%s>>\n", $stdout);
-            //($stderr) && printf("ERR: <<%s>>\n", $stderr);
-            if (($stdout && $this->expression) && ($source=='stdout'))
-                if (preg_match($this->expression, $stdout, $match))
-                    call_user_func($callback, $this->parseNames($match));
-            if (($stderr && $this->expression) && ($source=='stderr'))
-                if (preg_match($this->expression, $stderr, $match))
-                    call_user_func($callback, $this->parseNames($match));
-            usleep(10000);
-        }
-
-    }
-
-    protected function parseNames(array $match)
-    {
-        $ret = [];
-        foreach ($this->names as $k=>$index) {
-            $ret[$k] = $match[$index];
-        }
-        return $ret;
-    }
-
-}
-
 return new ExecutorPlugin();
+
diff --git a/plugins/transcode/plugin.php b/plugins/transcode/plugin.php
index 7920d8e..0a6ac14 100644
--- a/plugins/transcode/plugin.php
+++ b/plugins/transcode/plugin.php
@@ -16,7 +16,6 @@ class TranscodePlugin extends Plugin
     
     public function applyPreset(Preset $preset, Input $input, Output $output)
     {
-           
     }
 
 }
diff --git a/presets/executor/executor-scenesplit.yml b/presets/executor/executor-scenesplit.yml
new file mode 100644
index 0000000..c1acd0d
--- /dev/null
+++ b/presets/executor/executor-scenesplit.yml
@@ -0,0 +1,111 @@
+preset:
+    name: Split video on black frames (ffmpeg)
+    group: video
+    plugin: executor
+    props:
+        set:
+            scenes: { value:"{%uinput}.tmp", escape:true }
+        script:
+            analyze:
+                info: Finding scenes
+                exec: "ffmpeg -i {%input} -vf blackframe=amount=90:thresh=64 -f null -"
+                parse: { from: 'stderr', regex: '/^frame=[\s]*([0-9]+)\s/', frame:1 }
+                extract:
+                    # Write everything matching '[Parsed_blackframe' to file
+                    - { from: 'stderr', regex: '/^\[Parsed_blackframe/', write:'{%scenes}' }
+            split:
+                info: Splitting video
+                call: scenesplitter
+            cleanup:
+                info: Cleaning up
+                exec: "rm {%scenes}"
+        helpers:
+            scenesplitter: |
+                /*
+                * This is what the contents of %scenes look like:
+                *
+                * [Parsed_blackframe_0 @ 0x54f7960] frame:1312 pblack:100 pts:1313312 t:43.777067 type:P last_keyframe:1290
+                * [Parsed_blackframe_0 @ 0x54f7960] frame:1313 pblack:99 pts:1314313 t:43.810433 type:P last_keyframe:1290
+                * [Parsed_blackframe_0 @ 0x54f7960] frame:1314 pblack:99 pts:1315314 t:43.843800 type:P last_keyframe:1290
+                * [Parsed_blackframe_0 @ 0x54f7960] frame:1315 pblack:99 pts:1316315 t:43.877167 type:P last_keyframe:1290
+                * [Parsed_blackframe_0 @ 0x54f7960] frame:1316 pblack:99 pts:1317316 t:43.910533 type:P last_keyframe:1290
+                * [Parsed_blackframe_0 @ 0x54f7960] frame:1317 pblack:99 pts:1318317 t:43.943900 type:P last_keyframe:1290
+                * [Parsed_blackframe_0 @ 0x54f7960] frame:1318 pblack:99 pts:1319318 t:43.977267 type:P last_keyframe:1290
+                *
+                * Pick the split point as the last frame in a block with the highest percentage of black pixels, in this case
+                * it would be on the first line as it has pblack=100 for a single frame.
+                */
+
+                if (!file_exists({{%scenes}})) {
+                    return;
+                }
+
+                $frames = file({{%scenes}}, FILE_IGNORE_NEW_LINES);
+
+                $chunks = [];
+
+                // Split into chunks of blackish frames
+                $chunk = [];
+                $last_frame = 0;
+                foreach ($frames as $frame) {
+                    if (preg_match('/frame:(\d+) pblack:(\d+) pts:(\d+) t:([\d\.]+) type:(.+) last_keyframe:(\d+)$/', $frame, $info)) {
+                        list ($void, $_frame, $_pblack, $_pts, $_t, $_type, $_last_keyframe) = $info;
+                        if ($_frame > $last_frame+1) {
+                            // Add the chunk if its a new block
+                            if (count($chunk)>0) {
+                                $chunks[] = $chunk;
+                                $chunk = [];
+                            }
+                        }
+                        // Store chunk data
+                        $chunk[] = (object)[
+                            'frame' => $_frame,
+                            'pblack' => $_pblack,
+                            'time' => $_t
+                        ];
+                        $last_frame = $_frame;
+                    }
+                }
+                // Save the last chunk
+                if (count($chunk)>0) {
+                    $chunks[] = $chunk;
+                }
+
+                // Process chunks and find best split points
+                $last_time = 0;
+                $scenes = [];
+                foreach ($chunks as $chunk) {
+                    // Find split
+                    $best_black = 0; $best_time = 0;
+                    foreach ($chunk as $frame) {
+                        if ($frame->pblack >= $best_black) {
+                            $best_black = $frame->pblack;
+                            $best_time = $frame->time;
+                        }
+                    }
+                    $scenes[] = [ $last_time, $best_time ];
+                    $last_time = $best_time;
+                }
+
+                // Split it up
+                $input = {{%input}};
+                $output = {{%output}};
+
+                $outdir = dirname($output);
+                $outext = pathinfo($output, PATHINFO_EXTENSION);
+                $outfile = basename($output, ".{$outext}");
+
+                foreach ($scenes as $_index=>$scene) {
+                    list($_start,$_end) = $scene;
+                    $_duration = $_end-$_start;
+                    $cmdl = sprintf("ffmpeg -ss %s -i '%s' -t %s -codec copy '%s/%s-%d.%s' 2>/dev/null", 
+                        $_start, 
+                        $input, 
+                        $_duration, 
+                        $outdir, $outfile, $_index+1, $outext
+                    );
+                    printf("Exec: %s\n", $cmdl);
+                    exec($cmdl, $out, $ret);
+                }
+
+
diff --git a/src/Application.php b/src/Application.php
index a090f52..f3fc168 100644
--- a/src/Application.php
+++ b/src/Application.php
@@ -39,6 +39,16 @@ class Application
     private function loadPlugin($path)
     {
         $plugin = require_once $path."/plugin.php";
+        $plugin_class = get_class($plugin);
+        $plugin_ns = substr($plugin_class, 0, strrpos($plugin_class, '\\'));
+        spl_autoload_register(function ($cn) use ($plugin_ns, $path) {
+            if (strncmp($cn, $plugin_ns, strlen($plugin_ns))===0) {
+                $fn = $path . str_replace("\\",DIRECTORY_SEPARATOR,substr($cn,strlen($plugin_ns))) . ".php";
+                if (file_exists($fn)) {
+                    require_once $fn;
+                }
+            }
+        });
         $this->plugins[$plugin->getName()] = $plugin;
     }
 
diff --git a/src/Plugin.php b/src/Plugin.php
index d6597e6..6c57ce2 100644
--- a/src/Plugin.php
+++ b/src/Plugin.php
@@ -4,7 +4,7 @@ namespace VfxApply;
 
 class Plugin
 {
-    protected function createDialog($type, $title=null)
+    public function createDialog($type, $title=null)
     {
         switch ($type) {
             case DIALOG_PROGRESS:
@@ -14,7 +14,7 @@ class Plugin
         }
     }
 
-    protected function createProcess($command, callable $callback)
+    public function createProcess($command, callable $callback)
     {
         return new Process($command, $callback);
     }