Misc fixes and improvements

* Added request logging to com.noccy.apiclient
* Added plugin com.noccy.watcher
* Added pipe command and filter support
* Fixes and stubs
This commit is contained in:
Chris 2021-12-14 23:01:25 +01:00
parent 8cc1eac7a4
commit 30dfd4889b
22 changed files with 648 additions and 3 deletions

1
.gitignore vendored
View File

@ -4,3 +4,4 @@
/*.phar /*.phar
/src/version /src/version
/release /release
/.spark/plugins/*

View File

@ -2,3 +2,15 @@
Install by copying or symlinking into your `.spark/plugins` directory, or whatever Install by copying or symlinking into your `.spark/plugins` directory, or whatever
directory you have defined to preload from. directory you have defined to preload from.
## Plugins
- `com.noccy.apiclient`: Define and call web APIs. Initial support for HTTP, planned
support for WebSockets, XML-RPC and JSONRPC.
- `com.noccy.pdo`: Interact with PDO from the command line. Registers a custom
resource type, so database connections can be defined in advance to be available
to scripts and more.
- `com.noccy.watcher`: A plugin too watch files and invoke scripts or commands when
a modification is detected. This can be used to compile scss/less or to generate
other resources as files are changed.

View File

@ -6,6 +6,7 @@ use Spark\Commands\Command;
use SparkPlug; use SparkPlug;
use SparkPlug\Com\Noccy\ApiClient\Api\Method; use SparkPlug\Com\Noccy\ApiClient\Api\Method;
use SparkPlug\Com\Noccy\ApiClient\ApiClientPlugin; use SparkPlug\Com\Noccy\ApiClient\ApiClientPlugin;
use SparkPlug\Com\Noccy\ApiClient\Log\RequestData;
use SparkPlug\Com\Noccy\ApiClient\Request\RequestBuilder; use SparkPlug\Com\Noccy\ApiClient\Request\RequestBuilder;
use Symfony\Component\Console\Helper\Table; use Symfony\Component\Console\Helper\Table;
use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputArgument;
@ -129,6 +130,11 @@ class ApiRequestCommand extends Command
$output->writeln($separator); $output->writeln($separator);
$output->writeln(strlen($body)." bytes"); $output->writeln(strlen($body)." bytes");
$log = $plugin->getRequestLog("default");
$evt = RequestData::fromRequestResponse($request, $response, $input->getArgument('method'));
$log->append($evt);
$log->flush();
return Command::SUCCESS; return Command::SUCCESS;
} }
} }

View File

@ -3,6 +3,7 @@
namespace SparkPlug\Com\Noccy\ApiClient; namespace SparkPlug\Com\Noccy\ApiClient;
use SparkPlug; use SparkPlug;
use SparkPlug\Com\Noccy\ApiClient\Log\RequestLog;
class ApiClientPlugin extends SparkPlug class ApiClientPlugin extends SparkPlug
{ {
@ -104,6 +105,14 @@ class ApiClientPlugin extends SparkPlug
return array_keys($this->profiles); return array_keys($this->profiles);
} }
public function getRequestLog(string $name): RequestLog
{
$env = get_environment();
$logsDir = $env->getConfigDirectory() . "/api/logs/";
$log = new RequestLog($logsDir.$name.".json");
return $log;
}
} }
register_plugin("com.noccy.apiclient", new ApiClientPlugin); register_plugin("com.noccy.apiclient", new ApiClientPlugin);

View File

@ -18,8 +18,16 @@ class GitPlug extends SparkPlug
register_command(new GitIgnoreCommand()); register_command(new GitIgnoreCommand());
} }
public function getIgnoreList(bool $local=false)
{
$root = get_environment()->getProjectDirectory();
$file = $root . (!$local ? "/.gitignore" : "/.git/info/exclude");
return new IgnoreList($file);
}
} }
//if (file_exists(get_environment()->getConfigDirectory()."/maker.json")) { //if (file_exists(get_environment()->getConfigDirectory()."/maker.json")) {
register_plugin("com.noccy.git", new GitPlug()); register_plugin("com.noccy.git", new GitPlug());
require_once(__DIR__."/helpers.php");
//} //}

View File

@ -28,10 +28,20 @@ class PdoQueryCommand extends Command {
$stmt = $sourcePdo->query($query); $stmt = $sourcePdo->query($query);
$stmt->execute(); $stmt->execute();
$csv = $input->getOption("csv");
$table = new Table($output); $table = new Table($output);
$table->setStyle($box?"box":"compact"); $table->setStyle($box?"box":"compact");
$hasColumns = false; $hasColumns = false;
while ($row = $stmt->fetch(\PDO::FETCH_ASSOC)) { while ($row = $stmt->fetch(\PDO::FETCH_ASSOC)) {
if ($csv) {
$output->writeln(
join(",", array_map(function($v) {
return str_contains(',',$v) ? var_export($v,true) : $v;
}, $row))
);
continue;
}
if (!$hasColumns) { if (!$hasColumns) {
if ($vert) { if ($vert) {
$table->setHeaders([ "Field", "VarType", "Value" ]); $table->setHeaders([ "Field", "VarType", "Value" ]);
@ -76,6 +86,7 @@ class PdoQueryCommand extends Command {
$this->setName("pdo:query"); $this->setName("pdo:query");
$this->setDescription("Run a query against a defined PDO connection"); $this->setDescription("Run a query against a defined PDO connection");
$this->addOption("res", "r", InputOption::VALUE_REQUIRED, "Resource to query", "db"); $this->addOption("res", "r", InputOption::VALUE_REQUIRED, "Resource to query", "db");
$this->addOption("csv",null, InputOption::VALUE_NONE, "Output as CSV");
$this->addOption("vertical", "l", InputOption::VALUE_NONE, "Print result as rows instead of columns"); $this->addOption("vertical", "l", InputOption::VALUE_NONE, "Print result as rows instead of columns");
$this->addOption("box", null, InputOption::VALUE_NONE, "Use boxed table"); $this->addOption("box", null, InputOption::VALUE_NONE, "Use boxed table");
$this->addOption("unserialize", "u", InputOption::VALUE_NONE, "Attempt to unserialize serialized data"); $this->addOption("unserialize", "u", InputOption::VALUE_NONE, "Attempt to unserialize serialized data");

View File

@ -0,0 +1,84 @@
<?php
namespace SparkPlug\Com\Noccy\Pdo;
use Spark\Commands\Command;
use Symfony\Component\Console\Helper\Table;
use Symfony\Component\Console\Helper\TableSeparator;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class PdoStoreCommand extends Command {
protected function execute(InputInterface $input, OutputInterface $output)
{
$source = $input->getOption("res");
$sourcePdo = get_resource($source)->getPDO();
if (!$sourcePdo) {
$output->writeln("<error>Invalid resource: {$source}</>");
return Command::INVALID;
}
$box = $input->getOption('box');
$query = $input->getArgument('query');
$vert = $input->getOption("vertical");
$unserialize = $input->getOption("unserialize");
$stmt = $sourcePdo->query($query);
$stmt->execute();
$table = new Table($output);
$table->setStyle($box?"box":"compact");
$hasColumns = false;
while ($row = $stmt->fetch(\PDO::FETCH_ASSOC)) {
if (!$hasColumns) {
if ($vert) {
$table->setHeaders([ "Field", "VarType", "Value" ]);
} else {
$table->setHeaders(array_keys($row));
}
$hasColumns = true;
} else {
if ($vert) {
if ($box) {
$table->addRow(new TableSeparator());
} else {
$table->addRow(["","","-----"]);
}
}
}
if ($vert) {
foreach ($row as $k=>$v) {
$vv = $v;
if ($unserialize) {
$j = @json_decode($v);
$p = @unserialize($v);
if ($j) {
$v = $j;
$vv = json_encode($v, JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES);
} elseif ($p) {
$v = $p;
$vv = json_encode($p, JSON_PRETTY_PRINT|JSON_UNESCAPED_SLASHES);
}
}
$table->addRow([ $k, gettype($v), $vv ]);
}
} else {
$table->addRow($row);
}
}
$table->render();
return Command::SUCCESS;
}
protected function configure() {
$this->setName("pdo:store");
$this->setDescription("Store a query to recall later");
$this->addOption("res", "r", InputOption::VALUE_REQUIRED, "Resource to query", "db");
$this->addArgument("name", InputArgument::REQUIRED, "Query name");
$this->addArgument("query", InputArgument::REQUIRED, "SQL query to execute");
$this->addArgument("slots", InputArgument::IS_ARRAY|InputArgument::OPTIONAL, "Slots in the query string");
}
}

View File

@ -0,0 +1,33 @@
# PDO Plugin
## Usage
Storing queries:
$ spark pdo:store --res otherdb \ # store resource with query
"getuserid" \ # Query name
"select id from users where username=:username" \ # query
:username # slot
List stored queries:
$ spark pdo:store
Delete a stored query:
$ spark pdo:store --remove getuserid
Recalling queries:
$ spark pdo:query --recall getuserid username=bob
Direct query:
$ spark pdo:query "select * from users"
$ spark pdo:query --res otherdb "select * from users"
$ spark pdo:query --vertical "select * from user where id=:id" id=42
$ spark pdo:query --box --vertical "select name,value from config"

View File

@ -0,0 +1,56 @@
<?php
namespace SparkPlug\Com\Noccy\Watcher\Commands;
use Spark\Commands\Command;
use SparkPlug;
use SparkPlug\Com\Noccy\ApiClient\Api\Method;
use SparkPlug\Com\Noccy\ApiClient\ApiClientPlugin;
use SparkPlug\Com\Noccy\ApiClient\Log\RequestData;
use SparkPlug\Com\Noccy\ApiClient\Request\RequestBuilder;
use SparkPlug\Com\Noccy\Watcher\Rule;
use SparkPlug\Com\Noccy\Watcher\WatcherPlug;
use Symfony\Component\Console\Helper\Table;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class WatchCommand extends Command
{
protected function configure()
{
$this->setName("watch")
->setDescription("Watch files and take action when they are modified")
->addOption("interval", "N", InputOption::VALUE_REQUIRED, "Interval between polls", 5)
;
}
protected function execute(InputInterface $input, OutputInterface $output)
{
/** @var WatcherPlug $plugin */
$plugin = get_plugin('com.noccy.watcher');
$config = read_config('watchers.json');
$iv = max(1, (int)$input->getOption('interval'));
if (!($plugin && $config)) {
$output->writeln("<error>Missing or bad config file watchers.json?</>");
return Command::FAILURE;
}
$watcher = $plugin->getFileWatcher();
foreach ($config['watchers'] as $ruleconf) {
$rule = Rule::createFromConfig($ruleconf);
$watcher->addRule($rule);
}
while (true) {
$watcher->loop();
sleep($iv);
}
return Command::SUCCESS;
}
}

View File

@ -0,0 +1,53 @@
<?php // "name":"Watch files and act when they are changed", "author":"Noccy"
namespace SparkPlug\Com\Noccy\Watcher;
use Spark\Environment\ScriptRunner;
use SparkPlug\Com\Noccy\Watcher\Monitor\MonitorInterface;
use SparkPlug\Com\Noccy\Watcher\Monitor\MtimeMonitor;
use SparkPlug\Com\Noccy\Watcher\Monitor\InotifyMonitor;
class FileWatcher {
private MonitorInterface $monitor;
private ScriptRunner $scriptRunner;
private array $rules = [];
public function __construct()
{
if (extension_loaded('inotify')) {
$this->monitor = new MtimeMonitor();
//$this->monitor = new InotifyMonitor();
} else {
$this->monitor = new MtimeMonitor();
}
$this->scriptRunner = get_environment()->getScriptRunner();
}
public function addRule(Rule $rule)
{
if ($rule->getInitialTrigger()) {
$this->triggerRule($rule);
}
$this->rules[] = $rule;
$this->monitor->add($rule);
}
private function triggerRule(Rule $rule)
{
$actions = $rule->getActions();
$this->scriptRunner->evaluate($actions);
}
public function loop()
{
$this->monitor->loop();
$modified = $this->monitor->getModified();
foreach ($modified as $rule) {
$this->triggerRule($rule);
}
}
}

View File

@ -0,0 +1,28 @@
<?php
namespace SparkPlug\Com\Noccy\Watcher\Monitor;
use SparkPlug\Com\Noccy\Watcher\Rule;
interface MonitorInterface
{
/**
* Add a rule to be watched for changes
*/
public function add(Rule $rule);
/**
* Return a list of modified filenames
*/
public function getModified(): array;
/**
* Return a list of watched filenames
*/
public function getWatched(): array;
/**
* Called periodically to refresh monitors
*/
public function loop();
}

View File

@ -0,0 +1,77 @@
<?php
namespace SparkPlug\Com\Noccy\Watcher\Monitor;
use SparkPlug\Com\Noccy\Watcher\Rule;
class MtimeMonitor implements MonitorInterface
{
private array $rules = [];
private array $watched = [];
private array $modified = [];
/**
* {@inheritDoc}
*/
public function add(Rule $rule)
{
$this->rules[] = $rule;
}
/**
* {@inheritDoc}
*/
public function getModified(): array
{
$mod = $this->modified;
$this->modified = [];
return $mod;
}
/**
* {@inheritDoc}
*/
public function getWatched(): array
{
return [];
}
public function loop()
{
foreach ($this->rules as $rule) {
$this->checkRule($rule);
}
}
private function checkRule(Rule $rule)
{
clearstatcache();
$paths = $rule->getWatchedFiles();
$check = [];
foreach ($paths as $path) {
if (str_contains($path, '*')) {
$check = array_merge($check, glob($path));
} else {
$check[] = $path;
}
}
foreach ($check as $path) {
if (empty($this->watched[$path])) {
$this->watched[$path] = filemtime($path);
} else {
$mtime = filemtime($path);
if ($mtime > $this->watched[$path]) {
printf("* modified: %s (%s)\n", $path, $rule->getName());
$this->watched[$path] = $mtime;
if (!in_array($rule, $this->modified)) {
$this->modified[] = $rule;
}
}
}
}
}
}

View File

@ -0,0 +1,37 @@
# Watcher Plugin for Spark
Note: While the plugin currently supports wildcards, it does not scale well.
Keep the watched files to a minimum or increase the interval if you experience
issues.
## Usage
$ spark watch
## Installation
1. Install Spark with global plugins
2. Initialize your project: `spark init`
3. Enable the plugin with `spark plugin --enable com.noccy.watcher`
4. Configure your `.spark/watchers.json` file
## Configuration
*watchers.json*
```json
{
"watchers": [
{
"name": "name-of-rule",
"watch": [ "file1", "dir1/*" ],
"initial-trigger": true,
"actions": [
"@build"
]
}
]
}
```
The `initial-trigger` key controls whether the rule is triggered on startup.

View File

@ -0,0 +1,53 @@
<?php // "name":"Watch files and act when they are changed", "author":"Noccy"
namespace SparkPlug\Com\Noccy\Watcher;
class Rule
{
private array $filenames = [];
private array $actions = [];
private bool $initialTrigger = false;
private string $name;
public function __construct()
{
$this->name = "unnamed rule";
}
public static function createFromConfig(array $config)
{
$rule = new Rule();
$rule->filenames = (array)$config['watch'];
$rule->initialTrigger = ((bool)$config['initial-trigger'])??false;
$rule->actions = $config['actions']??[];
$rule->name = $config['name']??$rule->name;
return $rule;
}
public function getName(): string
{
return $this->name;
}
public function getInitialTrigger(): bool
{
return $this->initialTrigger;
}
public function getWatchedFiles(): array
{
return $this->filenames;
}
public function getActions(): array
{
return $this->actions;
}
}

View File

@ -0,0 +1,24 @@
<?php // "name":"Watch files and act when they are changed", "author":"Noccy"
namespace SparkPlug\Com\Noccy\Watcher;
class WatcherPlug extends \SparkPlug {
private ?FileWatcher $watcher = null;
public function load()
{
register_command(new Commands\WatchCommand());
//register_command(new PdoExecCommand());
}
public function getFileWatcher(): FileWatcher
{
if (!$this->watcher) {
$this->watcher = new FileWatcher();
}
return $this->watcher;
}
}
register_plugin("com.noccy.watcher", new WatcherPlug);

View File

@ -1,6 +1,7 @@
<?php <?php
use Spark\SparkApplication; use Spark\SparkApplication;
use Spark\Environment\Environment;
abstract class SparkPlug abstract class SparkPlug
{ {
@ -11,12 +12,22 @@ abstract class SparkPlug
return SparkApplication::$instance->getPluginManager()->getPlugin($name); return SparkApplication::$instance->getPluginManager()->getPlugin($name);
} }
function getResource(string $name) public function getResource(string $name)
{ {
return SparkApplication::$instance->getResourceManager()->getNamedResource($name); return SparkApplication::$instance->getResourceManager()->getNamedResource($name);
} }
function readConfig($file=null) public function getEnvironment(): Environment
{
return SparkApplication::$instance->getEnvironment();
}
public function getApplication(): SparkApplication
{
return SparkApplication::$instance;
}
public function readConfig($file=null)
{ {
if (!$file) return; if (!$file) return;
$abs = get_environment()->getConfigDirectory() . "/" . $file; $abs = get_environment()->getConfigDirectory() . "/" . $file;
@ -32,5 +43,9 @@ abstract class SparkPlug
return SparkApplication::$instance->getEnvironment()->getProjectDirectory(); return SparkApplication::$instance->getEnvironment()->getProjectDirectory();
} }
public function getConfigDirectory()
{
return SparkApplication::$instance->getEnvironment()->getConfigDirectory();
}
} }

View File

@ -81,3 +81,25 @@ function read_config($file=null) {
} }
return (array)json_decode(file_get_contents($abs), true); return (array)json_decode(file_get_contents($abs), true);
} }
// ------ Filters ------
$FILTERS = [];
function register_filter(string $name, callable $filter) {
global $FILTERS;
$FILTERS[$name] = $filter;
}
function get_registered_filters(): array
{
global $FILTERS;
return array_keys($FILTERS);
}
function get_filter(string $name): ?callable
{
global $FILTERS;
return $FILTERS[$name]??null;
}

View File

@ -0,0 +1,92 @@
<?php
namespace Spark\Commands;
use Symfony\Component\Console\Attribute\AsCommand;
use Spark\Commands\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
#[AsCommand(name:'pipe', description:'Filter or analyze data piped through the command')]
class PipeCommand extends Command
{
public static string $HelpText;
protected function configure()
{
$this->addOption("list-filters", null, InputOption::VALUE_NONE, "List the defined filters");
$this->addOption("fdin", null, InputOption::VALUE_REQUIRED, "Input fd, for reading from", 0);
$this->addOption("fdout", null, InputOption::VALUE_REQUIRED, "Output fd, for writing to", 1);
$this->addOption("fderr", null, InputOption::VALUE_REQUIRED, "Error fd, for progress report and status", 2);
$this->addArgument("filter", InputArgument::OPTIONAL, "Pipe filter");
$this->addArgument("args", InputArgument::OPTIONAL|InputArgument::IS_ARRAY, "Arguments to the script");
$this->registerDefaultFilters();
$this->setHelp(self::$HelpText);
}
private function registerDefaultFilters()
{
register_filter("base64encode", "base64_encode");
register_filter("base64decode", "base64_decode");
register_filter("passwordhash", function ($in) {
$trimmed = rtrim($in, "\n\r");
$hashed = password_hash($trimmed, PASSWORD_BCRYPT);
return str_replace($trimmed, $hashed, $in);
});
}
protected function execute(InputInterface $input, OutputInterface $output)
{
$env = $this->getEnvironment();
if ($input->getOption("list-filters")) {
$output->writeln(join(" ", get_registered_filters()));
return Command::SUCCESS;
}
$fdin = "php://fd/".$input->getOption("fdin");
$fdout = "php://fd/".$input->getOption("fdout");
$fderr = "php://fd/".$input->getOption("fderr");
$filtername = $input->getArgument("filter");
if ($filtername) {
$filter = get_filter($filtername);
} else {
$filter = null;
}
$fin = fopen($fdin, "rb");
$fout = fopen($fdout, "wb");
while (!feof($fin)) {
$buf = fgets($fin);
if (is_callable($filter)) $buf = $filter($buf);
fputs($fout, $buf);
}
return Command::SUCCESS;
}
}
PipeCommand::$HelpText = <<<HELP
The <info>pipe</> command is used to filter data, or to track piping of data.
\$ <comment>echo "mypassword" | spark pipe hashpassword > hashedpassword.txt</>
\$ <comment>cat file.sql | spark pipe sqlinfo | mysql</>
\$ <comment>cat input | spark pipe progress sizefrom=input | somecommand</>
<options=bold>Registering filters</>
To register a new filter, use the <info>register_filter</> helper function:
<comment>register_filter("myfilter", function (\$in) {
return strtolower(\$in);
});</>
The filter will be available like any built-in:
<comment>\$ cat file | spark pipe myfilter > outfile</>
HELP;

View File

@ -153,6 +153,25 @@ class Environment
} }
SparkApplication::$instance->getPluginManager()->initializePlugins(); SparkApplication::$instance->getPluginManager()->initializePlugins();
$this->loadResources();
}
private function loadResources()
{
$resourceFile = $this->getConfigDirectory() . "/resources.json";
if (!file_exists($resourceFile)) {
return;
}
$json = json_decode(
file_get_contents($resourceFile),
true
);
foreach ($json['resources'] as $name=>$uri) {
[$type, $uri] = explode("+", $uri, 2);
create_resource($name, $type, uri:$uri);
}
} }
public static function createFromDirectory(string|null $directory=null, bool $parents=false): Environment public static function createFromDirectory(string|null $directory=null, bool $parents=false): Environment

View File

@ -27,6 +27,9 @@ class ResourceManager
public function createNamedResource(string $name, string $type, array $options) public function createNamedResource(string $name, string $type, array $options)
{ {
if (array_key_exists($name, $this->namedResources)) {
fprintf(STDERR, "warning: Redefining named resource %s\n", $name);
}
$resource = $this->createResource($type, $options); $resource = $this->createResource($type, $options);
$this->namedResources[$name] = $resource; $this->namedResources[$name] = $resource;
return $resource; return $resource;

View File

@ -38,6 +38,7 @@ class SparkApplication extends Application
$this->add(new Commands\ResourcesCommand()); $this->add(new Commands\ResourcesCommand());
$this->add(new Commands\ReplCommand()); $this->add(new Commands\ReplCommand());
$this->add(new Commands\InitCommand()); $this->add(new Commands\InitCommand());
$this->add(new Commands\PipeCommand());
$this->get("list")->setHidden(true); $this->get("list")->setHidden(true);
$this->get("completion")->setHidden(true); $this->get("completion")->setHidden(true);

View File

@ -85,6 +85,7 @@ if ($doAliases) {
$file .= "alias sparksh=\"spark repl\"\n"; $file .= "alias sparksh=\"spark repl\"\n";
$file .= "alias sparker=\"spark run\"\n"; $file .= "alias sparker=\"spark run\"\n";
$file .= "alias sparkplug=\"spark plugins\"\n"; $file .= "alias sparkplug=\"spark plugins\"\n";
$file .= "alias sparkpipe=\"spark pipe\"\n";
file_put_contents(getenv("HOME")."/.bash_aliases.new", $file); file_put_contents(getenv("HOME")."/.bash_aliases.new", $file);
rename(getenv("HOME")."/.bash_aliases", getenv("HOME")."/.bash_aliases.bak"); rename(getenv("HOME")."/.bash_aliases", getenv("HOME")."/.bash_aliases.bak");
rename(getenv("HOME")."/.bash_aliases.new", getenv("HOME")."/.bash_aliases"); rename(getenv("HOME")."/.bash_aliases.new", getenv("HOME")."/.bash_aliases");