From 30dfd4889b67816dff481f2c094317f2d3855946 Mon Sep 17 00:00:00 2001 From: Christopher Vagnetoft Date: Tue, 14 Dec 2021 23:01:25 +0100 Subject: [PATCH] Misc fixes and improvements * Added request logging to com.noccy.apiclient * Added plugin com.noccy.watcher * Added pipe command and filter support * Fixes and stubs --- .gitignore | 1 + plugins/README.md | 12 +++ .../Commands/ApiRequestCommand.php | 6 ++ plugins/com.noccy.apiclient/sparkplug.php | 9 ++ plugins/com.noccy.git/sparkplug.php | 8 ++ plugins/com.noccy.pdo/PdoQueryCommand.php | 11 +++ plugins/com.noccy.pdo/PdoStoreCommand.php | 84 +++++++++++++++++ plugins/com.noccy.pdo/README.md | 33 +++++++ .../Commands/WatchCommand.php | 56 +++++++++++ plugins/com.noccy.watcher/FileWatcher.php | 53 +++++++++++ .../Monitor/MonitorInterface.php | 28 ++++++ .../Monitor/MtimeMonitor.php | 77 ++++++++++++++++ plugins/com.noccy.watcher/README.md | 37 ++++++++ plugins/com.noccy.watcher/Rule.php | 53 +++++++++++ plugins/com.noccy.watcher/sparkplug.php | 24 +++++ runtime/SparkPlug.php | 19 +++- runtime/functions.php | 22 +++++ src/Commands/PipeCommand.php | 92 +++++++++++++++++++ src/Environment/Environment.php | 19 ++++ src/Resource/ResourceManager.php | 5 +- src/SparkApplication.php | 1 + src/install | 1 + 22 files changed, 648 insertions(+), 3 deletions(-) create mode 100644 plugins/com.noccy.pdo/PdoStoreCommand.php create mode 100644 plugins/com.noccy.pdo/README.md create mode 100644 plugins/com.noccy.watcher/Commands/WatchCommand.php create mode 100644 plugins/com.noccy.watcher/FileWatcher.php create mode 100644 plugins/com.noccy.watcher/Monitor/MonitorInterface.php create mode 100644 plugins/com.noccy.watcher/Monitor/MtimeMonitor.php create mode 100644 plugins/com.noccy.watcher/README.md create mode 100644 plugins/com.noccy.watcher/Rule.php create mode 100644 plugins/com.noccy.watcher/sparkplug.php create mode 100644 src/Commands/PipeCommand.php diff --git a/.gitignore b/.gitignore index 9561957..c546183 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ /*.phar /src/version /release +/.spark/plugins/* diff --git a/plugins/README.md b/plugins/README.md index 7ce50bb..2e4113b 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -2,3 +2,15 @@ Install by copying or symlinking into your `.spark/plugins` directory, or whatever directory you have defined to preload from. + +## Plugins + +- `com.noccy.apiclient`: Define and call web APIs. Initial support for HTTP, planned + support for WebSockets, XML-RPC and JSONRPC. +- `com.noccy.pdo`: Interact with PDO from the command line. Registers a custom + resource type, so database connections can be defined in advance to be available + to scripts and more. +- `com.noccy.watcher`: A plugin too watch files and invoke scripts or commands when + a modification is detected. This can be used to compile scss/less or to generate + other resources as files are changed. + diff --git a/plugins/com.noccy.apiclient/Commands/ApiRequestCommand.php b/plugins/com.noccy.apiclient/Commands/ApiRequestCommand.php index d08671a..a1740ac 100644 --- a/plugins/com.noccy.apiclient/Commands/ApiRequestCommand.php +++ b/plugins/com.noccy.apiclient/Commands/ApiRequestCommand.php @@ -6,6 +6,7 @@ use Spark\Commands\Command; use SparkPlug; use SparkPlug\Com\Noccy\ApiClient\Api\Method; use SparkPlug\Com\Noccy\ApiClient\ApiClientPlugin; +use SparkPlug\Com\Noccy\ApiClient\Log\RequestData; use SparkPlug\Com\Noccy\ApiClient\Request\RequestBuilder; use Symfony\Component\Console\Helper\Table; use Symfony\Component\Console\Input\InputArgument; @@ -129,6 +130,11 @@ class ApiRequestCommand extends Command $output->writeln($separator); $output->writeln(strlen($body)." bytes"); + $log = $plugin->getRequestLog("default"); + $evt = RequestData::fromRequestResponse($request, $response, $input->getArgument('method')); + $log->append($evt); + $log->flush(); + return Command::SUCCESS; } } diff --git a/plugins/com.noccy.apiclient/sparkplug.php b/plugins/com.noccy.apiclient/sparkplug.php index e21cd9d..592fcec 100644 --- a/plugins/com.noccy.apiclient/sparkplug.php +++ b/plugins/com.noccy.apiclient/sparkplug.php @@ -3,6 +3,7 @@ namespace SparkPlug\Com\Noccy\ApiClient; use SparkPlug; +use SparkPlug\Com\Noccy\ApiClient\Log\RequestLog; class ApiClientPlugin extends SparkPlug { @@ -104,6 +105,14 @@ class ApiClientPlugin extends SparkPlug return array_keys($this->profiles); } + + public function getRequestLog(string $name): RequestLog + { + $env = get_environment(); + $logsDir = $env->getConfigDirectory() . "/api/logs/"; + $log = new RequestLog($logsDir.$name.".json"); + return $log; + } } register_plugin("com.noccy.apiclient", new ApiClientPlugin); diff --git a/plugins/com.noccy.git/sparkplug.php b/plugins/com.noccy.git/sparkplug.php index 56e05bf..f12b949 100644 --- a/plugins/com.noccy.git/sparkplug.php +++ b/plugins/com.noccy.git/sparkplug.php @@ -18,8 +18,16 @@ class GitPlug extends SparkPlug register_command(new GitIgnoreCommand()); } + + public function getIgnoreList(bool $local=false) + { + $root = get_environment()->getProjectDirectory(); + $file = $root . (!$local ? "/.gitignore" : "/.git/info/exclude"); + return new IgnoreList($file); + } } //if (file_exists(get_environment()->getConfigDirectory()."/maker.json")) { register_plugin("com.noccy.git", new GitPlug()); +require_once(__DIR__."/helpers.php"); //} diff --git a/plugins/com.noccy.pdo/PdoQueryCommand.php b/plugins/com.noccy.pdo/PdoQueryCommand.php index 3ce8d3e..0e1b697 100644 --- a/plugins/com.noccy.pdo/PdoQueryCommand.php +++ b/plugins/com.noccy.pdo/PdoQueryCommand.php @@ -28,10 +28,20 @@ class PdoQueryCommand extends Command { $stmt = $sourcePdo->query($query); $stmt->execute(); + $csv = $input->getOption("csv"); + $table = new Table($output); $table->setStyle($box?"box":"compact"); $hasColumns = false; while ($row = $stmt->fetch(\PDO::FETCH_ASSOC)) { + if ($csv) { + $output->writeln( + join(",", array_map(function($v) { + return str_contains(',',$v) ? var_export($v,true) : $v; + }, $row)) + ); + continue; + } if (!$hasColumns) { if ($vert) { $table->setHeaders([ "Field", "VarType", "Value" ]); @@ -76,6 +86,7 @@ class PdoQueryCommand extends Command { $this->setName("pdo:query"); $this->setDescription("Run a query against a defined PDO connection"); $this->addOption("res", "r", InputOption::VALUE_REQUIRED, "Resource to query", "db"); + $this->addOption("csv",null, InputOption::VALUE_NONE, "Output as CSV"); $this->addOption("vertical", "l", InputOption::VALUE_NONE, "Print result as rows instead of columns"); $this->addOption("box", null, InputOption::VALUE_NONE, "Use boxed table"); $this->addOption("unserialize", "u", InputOption::VALUE_NONE, "Attempt to unserialize serialized data"); diff --git a/plugins/com.noccy.pdo/PdoStoreCommand.php b/plugins/com.noccy.pdo/PdoStoreCommand.php new file mode 100644 index 0000000..a77518e --- /dev/null +++ b/plugins/com.noccy.pdo/PdoStoreCommand.php @@ -0,0 +1,84 @@ +getOption("res"); + $sourcePdo = get_resource($source)->getPDO(); + if (!$sourcePdo) { + $output->writeln("Invalid resource: {$source}"); + return Command::INVALID; + } + + $box = $input->getOption('box'); + $query = $input->getArgument('query'); + $vert = $input->getOption("vertical"); + $unserialize = $input->getOption("unserialize"); + + $stmt = $sourcePdo->query($query); + $stmt->execute(); + + $table = new Table($output); + $table->setStyle($box?"box":"compact"); + $hasColumns = false; + while ($row = $stmt->fetch(\PDO::FETCH_ASSOC)) { + if (!$hasColumns) { + if ($vert) { + $table->setHeaders([ "Field", "VarType", "Value" ]); + } else { + $table->setHeaders(array_keys($row)); + } + $hasColumns = true; + } else { + if ($vert) { + if ($box) { + $table->addRow(new TableSeparator()); + } else { + $table->addRow(["","","-----"]); + } + } + } + if ($vert) { + foreach ($row as $k=>$v) { + $vv = $v; + if ($unserialize) { + $j = @json_decode($v); + $p = @unserialize($v); + if ($j) { + $v = $j; + $vv = json_encode($v, JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES); + } elseif ($p) { + $v = $p; + $vv = json_encode($p, JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES); + } + } + $table->addRow([ $k, gettype($v), $vv ]); + } + } else { + $table->addRow($row); + } + } + $table->render(); + + return Command::SUCCESS; + } + protected function configure() { + $this->setName("pdo:store"); + $this->setDescription("Store a query to recall later"); + $this->addOption("res", "r", InputOption::VALUE_REQUIRED, "Resource to query", "db"); + $this->addArgument("name", InputArgument::REQUIRED, "Query name"); + $this->addArgument("query", InputArgument::REQUIRED, "SQL query to execute"); + $this->addArgument("slots", InputArgument::IS_ARRAY|InputArgument::OPTIONAL, "Slots in the query string"); + } +} + diff --git a/plugins/com.noccy.pdo/README.md b/plugins/com.noccy.pdo/README.md new file mode 100644 index 0000000..5b06442 --- /dev/null +++ b/plugins/com.noccy.pdo/README.md @@ -0,0 +1,33 @@ +# PDO Plugin + + + + +## Usage + +Storing queries: + + $ spark pdo:store --res otherdb \ # store resource with query + "getuserid" \ # Query name + "select id from users where username=:username" \ # query + :username # slot + +List stored queries: + + $ spark pdo:store + +Delete a stored query: + + $ spark pdo:store --remove getuserid + +Recalling queries: + + $ spark pdo:query --recall getuserid username=bob + +Direct query: + + $ spark pdo:query "select * from users" + $ spark pdo:query --res otherdb "select * from users" + $ spark pdo:query --vertical "select * from user where id=:id" id=42 + $ spark pdo:query --box --vertical "select name,value from config" + diff --git a/plugins/com.noccy.watcher/Commands/WatchCommand.php b/plugins/com.noccy.watcher/Commands/WatchCommand.php new file mode 100644 index 0000000..d4b84f1 --- /dev/null +++ b/plugins/com.noccy.watcher/Commands/WatchCommand.php @@ -0,0 +1,56 @@ +setName("watch") + ->setDescription("Watch files and take action when they are modified") + ->addOption("interval", "N", InputOption::VALUE_REQUIRED, "Interval between polls", 5) + ; + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + /** @var WatcherPlug $plugin */ + $plugin = get_plugin('com.noccy.watcher'); + $config = read_config('watchers.json'); + + $iv = max(1, (int)$input->getOption('interval')); + + if (!($plugin && $config)) { + $output->writeln("Missing or bad config file watchers.json?"); + return Command::FAILURE; + } + + $watcher = $plugin->getFileWatcher(); + + foreach ($config['watchers'] as $ruleconf) { + $rule = Rule::createFromConfig($ruleconf); + $watcher->addRule($rule); + } + + while (true) { + $watcher->loop(); + sleep($iv); + } + + return Command::SUCCESS; + } +} diff --git a/plugins/com.noccy.watcher/FileWatcher.php b/plugins/com.noccy.watcher/FileWatcher.php new file mode 100644 index 0000000..af6fd01 --- /dev/null +++ b/plugins/com.noccy.watcher/FileWatcher.php @@ -0,0 +1,53 @@ +monitor = new MtimeMonitor(); + //$this->monitor = new InotifyMonitor(); + } else { + $this->monitor = new MtimeMonitor(); + } + $this->scriptRunner = get_environment()->getScriptRunner(); + } + + public function addRule(Rule $rule) + { + if ($rule->getInitialTrigger()) { + $this->triggerRule($rule); + } + $this->rules[] = $rule; + $this->monitor->add($rule); + } + + private function triggerRule(Rule $rule) + { + $actions = $rule->getActions(); + $this->scriptRunner->evaluate($actions); + } + + public function loop() + { + $this->monitor->loop(); + $modified = $this->monitor->getModified(); + foreach ($modified as $rule) { + $this->triggerRule($rule); + } + } +} + diff --git a/plugins/com.noccy.watcher/Monitor/MonitorInterface.php b/plugins/com.noccy.watcher/Monitor/MonitorInterface.php new file mode 100644 index 0000000..5d7cb58 --- /dev/null +++ b/plugins/com.noccy.watcher/Monitor/MonitorInterface.php @@ -0,0 +1,28 @@ +rules[] = $rule; + } + + /** + * {@inheritDoc} + */ + public function getModified(): array + { + $mod = $this->modified; + $this->modified = []; + return $mod; + } + + /** + * {@inheritDoc} + */ + public function getWatched(): array + { + return []; + } + + public function loop() + { + foreach ($this->rules as $rule) { + $this->checkRule($rule); + } + } + + private function checkRule(Rule $rule) + { + clearstatcache(); + + $paths = $rule->getWatchedFiles(); + $check = []; + foreach ($paths as $path) { + if (str_contains($path, '*')) { + $check = array_merge($check, glob($path)); + } else { + $check[] = $path; + } + } + + foreach ($check as $path) { + if (empty($this->watched[$path])) { + $this->watched[$path] = filemtime($path); + } else { + $mtime = filemtime($path); + if ($mtime > $this->watched[$path]) { + printf("* modified: %s (%s)\n", $path, $rule->getName()); + $this->watched[$path] = $mtime; + if (!in_array($rule, $this->modified)) { + $this->modified[] = $rule; + } + } + } + } + } +} diff --git a/plugins/com.noccy.watcher/README.md b/plugins/com.noccy.watcher/README.md new file mode 100644 index 0000000..5f86351 --- /dev/null +++ b/plugins/com.noccy.watcher/README.md @@ -0,0 +1,37 @@ +# Watcher Plugin for Spark + +Note: While the plugin currently supports wildcards, it does not scale well. +Keep the watched files to a minimum or increase the interval if you experience +issues. + +## Usage + + $ spark watch + +## Installation + +1. Install Spark with global plugins +2. Initialize your project: `spark init` +3. Enable the plugin with `spark plugin --enable com.noccy.watcher` +4. Configure your `.spark/watchers.json` file + +## Configuration + +*watchers.json* + +```json +{ + "watchers": [ + { + "name": "name-of-rule", + "watch": [ "file1", "dir1/*" ], + "initial-trigger": true, + "actions": [ + "@build" + ] + } + ] +} +``` + +The `initial-trigger` key controls whether the rule is triggered on startup. diff --git a/plugins/com.noccy.watcher/Rule.php b/plugins/com.noccy.watcher/Rule.php new file mode 100644 index 0000000..ed6d516 --- /dev/null +++ b/plugins/com.noccy.watcher/Rule.php @@ -0,0 +1,53 @@ +name = "unnamed rule"; + } + + public static function createFromConfig(array $config) + { + $rule = new Rule(); + + $rule->filenames = (array)$config['watch']; + $rule->initialTrigger = ((bool)$config['initial-trigger'])??false; + $rule->actions = $config['actions']??[]; + $rule->name = $config['name']??$rule->name; + + return $rule; + } + + public function getName(): string + { + return $this->name; + } + + public function getInitialTrigger(): bool + { + return $this->initialTrigger; + } + + public function getWatchedFiles(): array + { + return $this->filenames; + } + + public function getActions(): array + { + return $this->actions; + } +} \ No newline at end of file diff --git a/plugins/com.noccy.watcher/sparkplug.php b/plugins/com.noccy.watcher/sparkplug.php new file mode 100644 index 0000000..b709aef --- /dev/null +++ b/plugins/com.noccy.watcher/sparkplug.php @@ -0,0 +1,24 @@ +watcher) { + $this->watcher = new FileWatcher(); + } + return $this->watcher; + } +} + +register_plugin("com.noccy.watcher", new WatcherPlug); diff --git a/runtime/SparkPlug.php b/runtime/SparkPlug.php index 4da3222..1230d73 100644 --- a/runtime/SparkPlug.php +++ b/runtime/SparkPlug.php @@ -1,6 +1,7 @@ getPluginManager()->getPlugin($name); } - function getResource(string $name) + public function getResource(string $name) { return SparkApplication::$instance->getResourceManager()->getNamedResource($name); } + + public function getEnvironment(): Environment + { + return SparkApplication::$instance->getEnvironment(); + } - function readConfig($file=null) + public function getApplication(): SparkApplication + { + return SparkApplication::$instance; + } + + public function readConfig($file=null) { if (!$file) return; $abs = get_environment()->getConfigDirectory() . "/" . $file; @@ -32,5 +43,9 @@ abstract class SparkPlug return SparkApplication::$instance->getEnvironment()->getProjectDirectory(); } + public function getConfigDirectory() + { + return SparkApplication::$instance->getEnvironment()->getConfigDirectory(); + } } \ No newline at end of file diff --git a/runtime/functions.php b/runtime/functions.php index f406dc2..e1d9927 100644 --- a/runtime/functions.php +++ b/runtime/functions.php @@ -81,3 +81,25 @@ function read_config($file=null) { } return (array)json_decode(file_get_contents($abs), true); } + + +// ------ Filters ------ + +$FILTERS = []; + +function register_filter(string $name, callable $filter) { + global $FILTERS; + $FILTERS[$name] = $filter; +} + +function get_registered_filters(): array +{ + global $FILTERS; + return array_keys($FILTERS); +} + +function get_filter(string $name): ?callable +{ + global $FILTERS; + return $FILTERS[$name]??null; +} \ No newline at end of file diff --git a/src/Commands/PipeCommand.php b/src/Commands/PipeCommand.php new file mode 100644 index 0000000..20fbf64 --- /dev/null +++ b/src/Commands/PipeCommand.php @@ -0,0 +1,92 @@ +addOption("list-filters", null, InputOption::VALUE_NONE, "List the defined filters"); + $this->addOption("fdin", null, InputOption::VALUE_REQUIRED, "Input fd, for reading from", 0); + $this->addOption("fdout", null, InputOption::VALUE_REQUIRED, "Output fd, for writing to", 1); + $this->addOption("fderr", null, InputOption::VALUE_REQUIRED, "Error fd, for progress report and status", 2); + $this->addArgument("filter", InputArgument::OPTIONAL, "Pipe filter"); + $this->addArgument("args", InputArgument::OPTIONAL|InputArgument::IS_ARRAY, "Arguments to the script"); + $this->registerDefaultFilters(); + $this->setHelp(self::$HelpText); + } + + private function registerDefaultFilters() + { + register_filter("base64encode", "base64_encode"); + register_filter("base64decode", "base64_decode"); + register_filter("passwordhash", function ($in) { + $trimmed = rtrim($in, "\n\r"); + $hashed = password_hash($trimmed, PASSWORD_BCRYPT); + return str_replace($trimmed, $hashed, $in); + }); + } + + protected function execute(InputInterface $input, OutputInterface $output) + { + $env = $this->getEnvironment(); + + if ($input->getOption("list-filters")) { + $output->writeln(join(" ", get_registered_filters())); + return Command::SUCCESS; + } + + $fdin = "php://fd/".$input->getOption("fdin"); + $fdout = "php://fd/".$input->getOption("fdout"); + $fderr = "php://fd/".$input->getOption("fderr"); + + $filtername = $input->getArgument("filter"); + if ($filtername) { + $filter = get_filter($filtername); + } else { + $filter = null; + } + + $fin = fopen($fdin, "rb"); + $fout = fopen($fdout, "wb"); + while (!feof($fin)) { + $buf = fgets($fin); + if (is_callable($filter)) $buf = $filter($buf); + fputs($fout, $buf); + } + + return Command::SUCCESS; + } +} + +PipeCommand::$HelpText = <<pipe command is used to filter data, or to track piping of data. + + \$ echo "mypassword" | spark pipe hashpassword > hashedpassword.txt + \$ cat file.sql | spark pipe sqlinfo | mysql + \$ cat input | spark pipe progress sizefrom=input | somecommand + +Registering filters + +To register a new filter, use the register_filter helper function: + + register_filter("myfilter", function (\$in) { + return strtolower(\$in); + }); + +The filter will be available like any built-in: + + \$ cat file | spark pipe myfilter > outfile + +HELP; diff --git a/src/Environment/Environment.php b/src/Environment/Environment.php index 092d5df..821e6ef 100644 --- a/src/Environment/Environment.php +++ b/src/Environment/Environment.php @@ -153,6 +153,25 @@ class Environment } SparkApplication::$instance->getPluginManager()->initializePlugins(); + + $this->loadResources(); + } + + private function loadResources() + { + $resourceFile = $this->getConfigDirectory() . "/resources.json"; + if (!file_exists($resourceFile)) { + return; + } + + $json = json_decode( + file_get_contents($resourceFile), + true + ); + foreach ($json['resources'] as $name=>$uri) { + [$type, $uri] = explode("+", $uri, 2); + create_resource($name, $type, uri:$uri); + } } public static function createFromDirectory(string|null $directory=null, bool $parents=false): Environment diff --git a/src/Resource/ResourceManager.php b/src/Resource/ResourceManager.php index 1adfa80..b93b40e 100644 --- a/src/Resource/ResourceManager.php +++ b/src/Resource/ResourceManager.php @@ -27,6 +27,9 @@ class ResourceManager public function createNamedResource(string $name, string $type, array $options) { + if (array_key_exists($name, $this->namedResources)) { + fprintf(STDERR, "warning: Redefining named resource %s\n", $name); + } $resource = $this->createResource($type, $options); $this->namedResources[$name] = $resource; return $resource; @@ -46,4 +49,4 @@ class ResourceManager { return $this->resourceTypes; } -} \ No newline at end of file +} diff --git a/src/SparkApplication.php b/src/SparkApplication.php index 29a8828..03ecdcd 100644 --- a/src/SparkApplication.php +++ b/src/SparkApplication.php @@ -38,6 +38,7 @@ class SparkApplication extends Application $this->add(new Commands\ResourcesCommand()); $this->add(new Commands\ReplCommand()); $this->add(new Commands\InitCommand()); + $this->add(new Commands\PipeCommand()); $this->get("list")->setHidden(true); $this->get("completion")->setHidden(true); diff --git a/src/install b/src/install index dfdb866..42afccd 100644 --- a/src/install +++ b/src/install @@ -85,6 +85,7 @@ if ($doAliases) { $file .= "alias sparksh=\"spark repl\"\n"; $file .= "alias sparker=\"spark run\"\n"; $file .= "alias sparkplug=\"spark plugins\"\n"; + $file .= "alias sparkpipe=\"spark pipe\"\n"; file_put_contents(getenv("HOME")."/.bash_aliases.new", $file); rename(getenv("HOME")."/.bash_aliases", getenv("HOME")."/.bash_aliases.bak"); rename(getenv("HOME")."/.bash_aliases.new", getenv("HOME")."/.bash_aliases");