Misc fixes and improvements

* Added request logging to com.noccy.apiclient
* Added plugin com.noccy.watcher
* Added pipe command and filter support
* Fixes and stubs

plugins/README.md

@@ -2,3 +2,15 @@
 
 Install by copying or symlinking into your `.spark/plugins` directory, or whatever
 directory you have defined to preload from.
+
+## Plugins
+
+- `com.noccy.apiclient`: Define and call web APIs. Initial support for HTTP, planned
+  support for WebSockets, XML-RPC and JSONRPC.
+- `com.noccy.pdo`: Interact with PDO from the command line. Registers a custom
+  resource type, so database connections can be defined in advance to be available
+  to scripts and more.
+- `com.noccy.watcher`: A plugin too watch files and invoke scripts or commands when
+  a modification is detected. This can be used to compile scss/less or to generate
+  other resources as files are changed.
+

plugins/com.noccy.apiclient/Commands/ApiRequestCommand.php

@@ -6,6 +6,7 @@ use Spark\Commands\Command;
 use SparkPlug;
 use SparkPlug\Com\Noccy\ApiClient\Api\Method;
 use SparkPlug\Com\Noccy\ApiClient\ApiClientPlugin;
+use SparkPlug\Com\Noccy\ApiClient\Log\RequestData;
 use SparkPlug\Com\Noccy\ApiClient\Request\RequestBuilder;
 use Symfony\Component\Console\Helper\Table;
 use Symfony\Component\Console\Input\InputArgument;
@@ -129,6 +130,11 @@ class ApiRequestCommand extends Command
         $output->writeln($separator);
         $output->writeln(strlen($body)." bytes");
 
+        $log = $plugin->getRequestLog("default");
+        $evt = RequestData::fromRequestResponse($request, $response, $input->getArgument('method'));
+        $log->append($evt);
+        $log->flush();
+
         return Command::SUCCESS;
     }
 }

plugins/com.noccy.apiclient/sparkplug.php

@@ -3,6 +3,7 @@
 namespace SparkPlug\Com\Noccy\ApiClient;
 
 use SparkPlug;
+use SparkPlug\Com\Noccy\ApiClient\Log\RequestLog;
 
 class ApiClientPlugin extends SparkPlug
 {
@@ -104,6 +105,14 @@ class ApiClientPlugin extends SparkPlug
 
         return array_keys($this->profiles);
     }
+
+    public function getRequestLog(string $name): RequestLog
+    {
+        $env = get_environment();
+        $logsDir = $env->getConfigDirectory() . "/api/logs/";
+        $log = new RequestLog($logsDir.$name.".json");
+        return $log;
+    }
 }
 
 register_plugin("com.noccy.apiclient", new ApiClientPlugin);

plugins/com.noccy.git/sparkplug.php

@@ -18,8 +18,16 @@ class GitPlug extends SparkPlug
         
         register_command(new GitIgnoreCommand());
     }
+
+    public function getIgnoreList(bool $local=false)
+    {
+        $root = get_environment()->getProjectDirectory();
+        $file = $root . (!$local ? "/.gitignore" : "/.git/info/exclude");
+        return new IgnoreList($file);
+    }
 }
 
 //if (file_exists(get_environment()->getConfigDirectory()."/maker.json")) {
 register_plugin("com.noccy.git", new GitPlug());
+require_once(__DIR__."/helpers.php");
 //}

plugins/com.noccy.pdo/PdoQueryCommand.php

@@ -28,10 +28,20 @@ class PdoQueryCommand extends Command {
         $stmt = $sourcePdo->query($query);
         $stmt->execute();
 
+        $csv = $input->getOption("csv");
+
         $table = new Table($output);
         $table->setStyle($box?"box":"compact");
         $hasColumns = false;
         while ($row = $stmt->fetch(\PDO::FETCH_ASSOC)) {
+            if ($csv) {
+                $output->writeln(
+                    join(",", array_map(function($v) {
+                        return str_contains(',',$v) ? var_export($v,true) : $v;
+                    }, $row))
+                );
+                continue;
+            }
             if (!$hasColumns) {
                 if ($vert) {
                     $table->setHeaders([ "Field", "VarType", "Value" ]);
@@ -76,6 +86,7 @@ class PdoQueryCommand extends Command {
         $this->setName("pdo:query");
         $this->setDescription("Run a query against a defined PDO connection");
         $this->addOption("res", "r", InputOption::VALUE_REQUIRED, "Resource to query", "db");
+        $this->addOption("csv",null, InputOption::VALUE_NONE, "Output as CSV");
         $this->addOption("vertical", "l", InputOption::VALUE_NONE, "Print result as rows instead of columns");
         $this->addOption("box", null, InputOption::VALUE_NONE, "Use boxed table");
         $this->addOption("unserialize", "u", InputOption::VALUE_NONE, "Attempt to unserialize serialized data");

plugins/com.noccy.pdo/PdoStoreCommand.php

@@ -0,0 +1,84 @@
+<?php
+
+namespace SparkPlug\Com\Noccy\Pdo;
+
+use Spark\Commands\Command;
+use Symfony\Component\Console\Helper\Table;
+use Symfony\Component\Console\Helper\TableSeparator;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class PdoStoreCommand extends Command {
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $source = $input->getOption("res");
+        $sourcePdo = get_resource($source)->getPDO();
+        if (!$sourcePdo) {
+            $output->writeln("<error>Invalid resource: {$source}</>");
+            return Command::INVALID;
+        }
+
+        $box = $input->getOption('box');
+        $query = $input->getArgument('query');
+        $vert = $input->getOption("vertical");
+        $unserialize = $input->getOption("unserialize");
+
+        $stmt = $sourcePdo->query($query);
+        $stmt->execute();
+
+        $table = new Table($output);
+        $table->setStyle($box?"box":"compact");
+        $hasColumns = false;
+        while ($row = $stmt->fetch(\PDO::FETCH_ASSOC)) {
+            if (!$hasColumns) {
+                if ($vert) {
+                    $table->setHeaders([ "Field", "VarType", "Value" ]);
+                } else {
+                    $table->setHeaders(array_keys($row));
+                }
+                $hasColumns = true;
+            } else {
+                if ($vert) {
+                    if ($box) {
+                        $table->addRow(new TableSeparator());
+                    } else {
+                        $table->addRow(["","","-----"]);
+                    }
+                }
+            }
+            if ($vert) {
+                foreach ($row as $k=>$v) {
+                    $vv = $v;
+                    if ($unserialize) {
+                        $j = @json_decode($v);
+                        $p = @unserialize($v);
+                        if ($j) {
+                            $v = $j;
+                            $vv = json_encode($v, JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES);
+                        } elseif ($p) {
+                            $v = $p;
+                            $vv = json_encode($p, JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES);
+                        }
+                    }
+                    $table->addRow([ $k, gettype($v), $vv ]);
+                }
+            } else {
+                $table->addRow($row);
+            }
+        }
+        $table->render();
+
+        return Command::SUCCESS;
+    }
+    protected function configure() {
+        $this->setName("pdo:store");
+        $this->setDescription("Store a query to recall later");
+        $this->addOption("res", "r", InputOption::VALUE_REQUIRED, "Resource to query", "db");
+        $this->addArgument("name", InputArgument::REQUIRED, "Query name");
+        $this->addArgument("query", InputArgument::REQUIRED, "SQL query to execute");
+        $this->addArgument("slots", InputArgument::IS_ARRAY|InputArgument::OPTIONAL, "Slots in the query string");
+    }
+}
+

plugins/com.noccy.pdo/README.md

@@ -0,0 +1,33 @@
+# PDO Plugin
+
+
+
+
+## Usage
+
+Storing queries:
+
+    $ spark pdo:store --res otherdb \  # store resource with query
+        "getuserid" \  # Query name
+        "select id from users where username=:username" \  # query
+        :username  # slot
+
+List stored queries:
+
+    $ spark pdo:store
+
+Delete a stored query:
+
+    $ spark pdo:store --remove getuserid
+
+Recalling queries:
+
+    $ spark pdo:query --recall getuserid username=bob
+
+Direct query:
+
+    $ spark pdo:query "select * from users"
+    $ spark pdo:query --res otherdb "select * from users"
+    $ spark pdo:query --vertical "select * from user where id=:id" id=42
+    $ spark pdo:query --box --vertical "select name,value from config"
+

plugins/com.noccy.watcher/Commands/WatchCommand.php

@@ -0,0 +1,56 @@
+<?php
+
+namespace SparkPlug\Com\Noccy\Watcher\Commands;
+
+use Spark\Commands\Command;
+use SparkPlug;
+use SparkPlug\Com\Noccy\ApiClient\Api\Method;
+use SparkPlug\Com\Noccy\ApiClient\ApiClientPlugin;
+use SparkPlug\Com\Noccy\ApiClient\Log\RequestData;
+use SparkPlug\Com\Noccy\ApiClient\Request\RequestBuilder;
+use SparkPlug\Com\Noccy\Watcher\Rule;
+use SparkPlug\Com\Noccy\Watcher\WatcherPlug;
+use Symfony\Component\Console\Helper\Table;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class WatchCommand extends Command
+{
+    protected function configure()
+    {
+        $this->setName("watch")
+            ->setDescription("Watch files and take action when they are modified")
+            ->addOption("interval", "N", InputOption::VALUE_REQUIRED, "Interval between polls", 5)
+            ;
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        /** @var WatcherPlug $plugin */
+        $plugin = get_plugin('com.noccy.watcher');
+        $config = read_config('watchers.json');
+
+        $iv = max(1, (int)$input->getOption('interval'));
+
+        if (!($plugin && $config)) {
+            $output->writeln("<error>Missing or bad config file watchers.json?</>");
+            return Command::FAILURE;
+        }
+
+        $watcher = $plugin->getFileWatcher();
+
+        foreach ($config['watchers'] as $ruleconf) {
+            $rule = Rule::createFromConfig($ruleconf);
+            $watcher->addRule($rule);
+        }
+
+        while (true) {
+            $watcher->loop();
+            sleep($iv);
+        }
+
+        return Command::SUCCESS;
+    }
+}

plugins/com.noccy.watcher/FileWatcher.php

@@ -0,0 +1,53 @@
+<?php // "name":"Watch files and act when they are changed", "author":"Noccy"
+
+namespace SparkPlug\Com\Noccy\Watcher;
+
+use Spark\Environment\ScriptRunner;
+use SparkPlug\Com\Noccy\Watcher\Monitor\MonitorInterface;
+use SparkPlug\Com\Noccy\Watcher\Monitor\MtimeMonitor;
+use SparkPlug\Com\Noccy\Watcher\Monitor\InotifyMonitor;
+
+class FileWatcher {
+
+    private MonitorInterface $monitor;
+
+    private ScriptRunner $scriptRunner;
+
+    private array $rules = [];
+
+    public function __construct()
+    {
+        if (extension_loaded('inotify')) {
+            $this->monitor = new MtimeMonitor();
+            //$this->monitor = new InotifyMonitor();
+        } else {
+            $this->monitor = new MtimeMonitor();
+        }
+        $this->scriptRunner = get_environment()->getScriptRunner();
+    }
+
+    public function addRule(Rule $rule)
+    {
+        if ($rule->getInitialTrigger()) {
+            $this->triggerRule($rule);
+        }
+        $this->rules[] = $rule;
+        $this->monitor->add($rule);
+    }
+
+    private function triggerRule(Rule $rule) 
+    {
+        $actions = $rule->getActions();
+        $this->scriptRunner->evaluate($actions);
+    }
+
+    public function loop()
+    {
+        $this->monitor->loop();
+        $modified = $this->monitor->getModified();
+        foreach ($modified as $rule) {
+            $this->triggerRule($rule);
+        }
+    }
+}
+

plugins/com.noccy.watcher/Monitor/MonitorInterface.php

@@ -0,0 +1,28 @@
+<?php
+
+namespace SparkPlug\Com\Noccy\Watcher\Monitor;
+
+use SparkPlug\Com\Noccy\Watcher\Rule;
+
+interface MonitorInterface
+{
+    /**
+     * Add a rule to be watched for changes
+     */
+    public function add(Rule $rule);
+
+    /**
+     * Return a list of modified filenames
+     */
+    public function getModified(): array;
+
+    /**
+     * Return a list of watched filenames
+     */
+    public function getWatched(): array;
+
+    /**
+     * Called periodically to refresh monitors
+     */
+    public function loop();
+}

plugins/com.noccy.watcher/Monitor/MtimeMonitor.php

@@ -0,0 +1,77 @@
+<?php
+
+namespace SparkPlug\Com\Noccy\Watcher\Monitor;
+
+use SparkPlug\Com\Noccy\Watcher\Rule;
+
+class MtimeMonitor implements MonitorInterface
+{
+    private array $rules = [];
+
+    private array $watched = [];
+
+    private array $modified = [];
+
+    /**
+     * {@inheritDoc}
+     */
+    public function add(Rule $rule)
+    {
+        $this->rules[] = $rule;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public function getModified(): array
+    {
+        $mod = $this->modified;
+        $this->modified = [];
+        return $mod;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public function getWatched(): array
+    {
+        return [];        
+    }
+
+    public function loop()
+    {
+        foreach ($this->rules as $rule) {
+            $this->checkRule($rule);
+        }
+    }
+
+    private function checkRule(Rule $rule)
+    {
+        clearstatcache();
+
+        $paths = $rule->getWatchedFiles();
+        $check = [];
+        foreach ($paths as $path) {
+            if (str_contains($path, '*')) {
+                $check = array_merge($check, glob($path));
+            } else {
+                $check[] = $path;
+            }
+        }
+
+        foreach ($check as $path) {
+            if (empty($this->watched[$path])) {
+                $this->watched[$path] = filemtime($path);
+            } else {
+                $mtime = filemtime($path);
+                if ($mtime > $this->watched[$path]) {
+                    printf("* modified: %s (%s)\n", $path, $rule->getName());
+                    $this->watched[$path] = $mtime;
+                    if (!in_array($rule, $this->modified)) {
+                        $this->modified[] = $rule;
+                    }
+                }
+            }
+        }
+    }
+}

plugins/com.noccy.watcher/README.md

@@ -0,0 +1,37 @@
+# Watcher Plugin for Spark
+
+Note: While the plugin currently supports wildcards, it does not scale well.
+Keep the watched files to a minimum or increase the interval if you experience
+issues.
+
+## Usage
+
+    $ spark watch
+
+## Installation
+
+1. Install Spark with global plugins
+2. Initialize your project: `spark init`
+3. Enable the plugin with `spark plugin --enable com.noccy.watcher`
+4. Configure your `.spark/watchers.json` file
+
+## Configuration
+
+*watchers.json*
+
+```json
+{
+  "watchers": [
+    {
+      "name": "name-of-rule",
+      "watch": [ "file1", "dir1/*" ],
+      "initial-trigger": true,
+      "actions": [
+        "@build"
+      ]
+    }
+  ]
+}
+```
+
+The `initial-trigger` key controls whether the rule is triggered on startup.

plugins/com.noccy.watcher/Rule.php

@@ -0,0 +1,53 @@
+<?php // "name":"Watch files and act when they are changed", "author":"Noccy"
+
+namespace SparkPlug\Com\Noccy\Watcher;
+
+
+class Rule
+{
+
+    private array $filenames = [];
+
+    private array $actions = [];
+
+    private bool $initialTrigger = false;
+
+    private string $name;
+
+    public function __construct()
+    {
+        $this->name = "unnamed rule";
+    }
+
+    public static function createFromConfig(array $config)
+    {
+        $rule = new Rule();
+
+        $rule->filenames = (array)$config['watch'];
+        $rule->initialTrigger = ((bool)$config['initial-trigger'])??false;
+        $rule->actions = $config['actions']??[];
+        $rule->name = $config['name']??$rule->name;
+
+        return $rule;
+    }
+
+    public function getName(): string
+    {
+        return $this->name;
+    }
+
+    public function getInitialTrigger(): bool
+    {
+        return $this->initialTrigger;
+    }
+
+    public function getWatchedFiles(): array
+    {
+        return $this->filenames;
+    }
+
+    public function getActions(): array
+    {
+        return $this->actions;
+    }
+}

plugins/com.noccy.watcher/sparkplug.php

@@ -0,0 +1,24 @@
+<?php // "name":"Watch files and act when they are changed", "author":"Noccy"
+
+namespace SparkPlug\Com\Noccy\Watcher;
+
+class WatcherPlug extends \SparkPlug {
+
+    private ?FileWatcher $watcher = null;
+
+    public function load()
+    {
+        register_command(new Commands\WatchCommand());
+        //register_command(new PdoExecCommand());
+    }
+
+    public function getFileWatcher(): FileWatcher
+    {
+        if (!$this->watcher) {
+            $this->watcher = new FileWatcher();
+        }
+        return $this->watcher;
+    }
+}
+
+register_plugin("com.noccy.watcher", new WatcherPlug);