Merge pull request #705 from cosmocode/lazy-search

Lazy search
This commit is contained in:
Andreas Gohr
2024-02-14 10:31:03 +01:00
committed by GitHub
18 changed files with 259 additions and 196 deletions

View File

@ -93,7 +93,7 @@ class AccessTableDataReplacementTest extends StructTest
$search = new meta\SearchConfig($actual_config);
list(, $opts) = $search->getSQL();
$result = $search->execute();
$result = $search->getRows();
$this->assertEquals(['page1', 'page2'], $opts, '$STRUCT.table.col$ should not require table to be selected');
$this->assertEquals('data of page1', $result[0][1]->getValue());
@ -114,7 +114,7 @@ class AccessTableDataReplacementTest extends StructTest
$actual_config = $configParser->getConfig();
$search = new meta\SearchConfig($actual_config);
$result = $search->execute();
$result = $search->getRows();
$this->assertEquals(0, count($result), 'if no pages a given, then none should be shown');
}

View File

@ -6,8 +6,6 @@ use dokuwiki\plugin\struct\meta\AccessTablePage;
use dokuwiki\plugin\struct\meta\ConfigParser;
use dokuwiki\plugin\struct\meta\PageMeta;
use dokuwiki\plugin\struct\test\mock\AccessTable as MockAccessTableAlias;
use dokuwiki\plugin\struct\test\mock\AggregationEditorTable as MockAggregationEditorTableAlias;
use dokuwiki\plugin\struct\test\mock\AggregationTable as MockAggregationTableAlias;
use dokuwiki\plugin\struct\test\mock\SearchConfig as MockSearchConfigAlias;
/**
@ -181,8 +179,7 @@ class AggregationResultsTest extends StructTest
if ($filters) $config['filter'][] = $filters;
$search = new MockSearchConfigAlias($config);
$table = new MockAggregationTableAlias($id, 'xhtml', new \Doku_Renderer_xhtml(), $search);
return $table->getResult();
return $search->getRows();
}
/**
@ -211,8 +208,7 @@ class AggregationResultsTest extends StructTest
if ($filters) $config['filter'][] = $filters;
$search = new MockSearchConfigAlias($config);
$table = new MockAggregationEditorTableAlias($id, 'xhtml', new \Doku_Renderer_xhtml(), $search);
return $table->getResult();
return $search->getRows();
}
protected function prepareLookup()

View File

@ -16,6 +16,9 @@ class SearchTest extends StructTest
public function setUp(): void
{
// workaround for recent GitHub disk I/O errors
parent::setUpBeforeClass();
parent::setUp();
$this->loadSchemaJSON('schema1');
@ -103,7 +106,7 @@ class SearchTest extends StructTest
$search->addColumn('second');
/** @var meta\Value[][] $result */
$result = $search->execute();
$result = $search->getRows();
$this->assertCount(2, $result, 'result rows');
$this->assertCount(3, $result[0], 'result columns');
@ -122,7 +125,7 @@ class SearchTest extends StructTest
$search->addColumn('second');
/** @var meta\Value[][] $result */
$result = $search->execute();
$result = $search->getRows();
$this->assertCount(2, $result, 'result rows');
$this->assertCount(3, $result[0], 'result columns');
@ -142,7 +145,7 @@ class SearchTest extends StructTest
$search->addColumn('second');
/** @var meta\Value[][] $result */
$result = $search->execute();
$result = $search->getRows();
$this->assertCount(0, $result, 'result rows');
}
@ -158,7 +161,7 @@ class SearchTest extends StructTest
$search->addColumn('second');
/** @var meta\Value[][] $result */
$result = $search->execute();
$result = $search->getRows();
$this->assertCount(2, $result, 'result rows');
$this->assertCount(4, $result[0], 'result columns');
@ -185,7 +188,7 @@ class SearchTest extends StructTest
$search->addColumn('second');
/** @var meta\Value[][] $result */
$result = $search->execute();
$result = $search->getRows();
$expected_time = dformat(filemtime(wikiFN('page01')), '%Y-%m-%d %H:%M:%S');
@ -214,7 +217,7 @@ class SearchTest extends StructTest
$search->addColumn('second');
/** @var meta\Value[][] $result */
$result = $search->execute();
$result = $search->getRows();
$this->assertCount(2, $result, 'result rows');
$this->assertCount(4, $result[0], 'result columns');
@ -270,7 +273,7 @@ class SearchTest extends StructTest
$search->addFilter('second', '%sec%', '~', 'AND');
$search->addFilter('first', '%rst%', '~', 'AND');
$result = $search->execute();
$result = $search->getRows();
$count = $search->getCount();
$this->assertEquals(1, $count, 'result count');
@ -280,40 +283,25 @@ class SearchTest extends StructTest
// sort by multi-column
$search->addSort('second');
$this->assertCount(2, $search->sortby);
$result = $search->execute();
$result = $search->getRows();
$count = $search->getCount();
$this->assertEquals(1, $count, 'result count');
$this->assertCount(1, $result, 'result rows');
$this->assertCount(6, $result[0], 'result columns');
/*
{#debugging
list($sql, $opts) = $search->getSQL();
print "\n";
print_r($sql);
print "\n";
print_r($opts);
print "\n";
#print_r($result);
}
*/
}
public function test_ranges()
{
$search = new mock\Search();
$search->addSchema('schema2');
$search->addColumn('%pageid%');
$search->addColumn('afirst');
$search->addColumn('asecond');
$search->addFilter('%pageid%', '%ag%', '~', 'AND');
$search->addSort('%pageid%', false);
/** @var meta\Value[][] $result */
$result = $search->execute();
$result = $search->getRows();
$count = $search->getCount();
// check result dimensions
@ -326,9 +314,19 @@ class SearchTest extends StructTest
$this->assertEquals('page19', $result[1][0]->getValue());
$this->assertEquals('page18', $result[2][0]->getValue());
// now add limit
// now with limit
// new search object because result is fetched only once
$search = new mock\Search();
$search->addSchema('schema2');
$search->addColumn('%pageid%');
$search->addColumn('afirst');
$search->addColumn('asecond');
$search->addFilter('%pageid%', '%ag%', '~', 'AND');
$search->addSort('%pageid%', false);
$search->setLimit(5);
$result = $search->execute();
/** @var meta\Value[][] $result */
$result = $search->getRows();
$count = $search->getCount();
// check result dimensions
@ -340,8 +338,17 @@ class SearchTest extends StructTest
$this->assertEquals('page16', $result[4][0]->getValue());
// now add offset
// again a new object
$search = new mock\Search();
$search->addSchema('schema2');
$search->addColumn('%pageid%');
$search->addColumn('afirst');
$search->addColumn('asecond');
$search->addFilter('%pageid%', '%ag%', '~', 'AND');
$search->addSort('%pageid%', false);
$search->setLimit(5);
$search->setOffset(5);
$result = $search->execute();
$result = $search->getRows();
$count = $search->getCount();
// check result dimensions

View File

@ -1,13 +0,0 @@
<?php
namespace dokuwiki\plugin\struct\test\mock;
use dokuwiki\plugin\struct\meta;
class AggregationTable extends meta\AggregationTable
{
public function getResult()
{
return $this->result;
}
}

View File

@ -191,7 +191,7 @@ class DecimalTest extends StructTest
$search->addColumn('field');
$search->addSort('field', true);
/** @var Value[][] $result */
$result = $search->execute();
$result = $search->getRows();
$this->assertEquals(4, count($result));
$this->assertEquals('page4', $result[0][0]->getValue());
@ -216,7 +216,7 @@ class DecimalTest extends StructTest
$search->addFilter('field', '800', '>', 'AND');
$search->addSort('field', true);
/** @var Value[][] $result */
$result = $search->execute();
$result = $search->getRows();
$this->assertEquals(3, count($result));
$this->assertEquals('page3', $result[0][0]->getValue());

View File

@ -58,7 +58,7 @@ class PageTest extends StructTest
$search->addColumn('singletitle');
$search->addSort('singletitle', true);
/** @var Value[][] $result */
$result = $search->execute();
$result = $search->getRows();
$this->assertEquals(3, count($result));
$this->assertEquals('test3', $result[0][0]->getValue());
@ -98,7 +98,7 @@ class PageTest extends StructTest
$search->addColumn('multititle');
/** @var Value[][] $result */
$result = $search->execute();
$result = $search->getRows();
// no titles:
$this->assertEquals('wiki:dokuwiki', $result[0][0]->getValue());
@ -127,28 +127,28 @@ class PageTest extends StructTest
// search single with title
$single = clone $search;
$single->addFilter('singletitle', 'Overview', '*~', 'AND');
$result = $single->execute();
$result = $single->getRows();
$this->assertTrue(is_array($result));
$this->assertEquals(1, count($result));
// search multi with title
$multi = clone $search;
$multi->addFilter('multititle', 'Foobar', '*~', 'AND');
$result = $multi->execute();
$result = $multi->getRows();
$this->assertTrue(is_array($result));
$this->assertEquals(1, count($result));
// search single with page
$single = clone $search;
$single->addFilter('singletitle', 'wiki:dokuwiki', '*~', 'AND');
$result = $single->execute();
$result = $single->getRows();
$this->assertTrue(is_array($result));
$this->assertEquals(1, count($result));
// search multi with page
$multi = clone $search;
$multi->addFilter('multititle', 'welcome', '*~', 'AND');
$result = $multi->execute();
$result = $multi->getRows();
$this->assertTrue(is_array($result));
$this->assertEquals(1, count($result));
}

View File

@ -103,7 +103,7 @@ class action_plugin_struct_bureaucracy extends ActionPlugin
$search = new Search();
$search->addSchema($config['schema']);
$search->addColumn($config['field']);
$result = $search->execute();
$result = $search->getRows();
$pids = $search->getPids();
$rids = $search->getRids();

View File

@ -24,12 +24,6 @@ abstract class Aggregation
/** @var Column[] the list of columns to be displayed */
protected $columns;
/** @var Value[][] the search result */
protected $result;
/** @var int number of all results */
protected $resultCount;
/** @var string usually a div, but AggregationValue needs to be wrapped in a span */
protected $tagName = 'div';
@ -62,8 +56,6 @@ abstract class Aggregation
$this->searchConfig = $searchConfig;
$this->data = $searchConfig->getConf();
$this->columns = $searchConfig->getColumns();
$this->result = $this->searchConfig->execute();
$this->resultCount = $this->searchConfig->getCount();
$this->helper = plugin_load('helper', 'struct_config');
}

View File

@ -10,30 +10,19 @@ class AggregationCloud extends Aggregation
/** @var int */
protected $min;
/**
* Initialize the Aggregation renderer and executes the search
*
* You need to call @param string $id
* @param string $mode
* @param \Doku_Renderer $renderer
* @param SearchConfig $searchConfig
* @see render() on the resulting object.
*
*/
public function __construct($id, $mode, \Doku_Renderer $renderer, SearchCloud $searchConfig)
{
parent::__construct($id, $mode, $renderer, $searchConfig);
$this->max = $this->result[0]['count'];
$this->min = end($this->result)['count'];
}
/** @inheritdoc */
public function render($showNotFound = false)
{
$this->sortResults();
if ($this->mode !== 'xhtml') return;
$rows = $this->searchConfig->getRows();
$this->max = $rows[0]['count'];
$this->min = end($rows)['count'];
$this->sortResults($rows);
$this->startList();
foreach ($this->result as $result) {
foreach ($rows as $result) {
$this->renderTag($result);
}
$this->finishList();
@ -110,9 +99,9 @@ class AggregationCloud extends Aggregation
/**
* Sort the list of results
*/
protected function sortResults()
protected function sortResults(&$rows)
{
usort($this->result, function ($a, $b) {
usort($rows, function ($a, $b) {
$asort = $a['tag']->getColumn()->getType()->getSortString($a['tag']);
$bsort = $b['tag']->getColumn()->getType()->getSortString($b['tag']);
if ($asort < $bsort) {

View File

@ -70,7 +70,7 @@ class AggregationEditorTable extends AggregationTable
$this->renderer->table_open();
$this->renderer->doc = '';
$this->renderResultRow(0, $this->result[0]);
$this->renderResultRow(0, $this->searchConfig->getRows()[0]);
return $this->renderer->doc;
}
}

View File

@ -22,8 +22,8 @@ class AggregationList extends Aggregation
/** @inheritdoc */
public function render($showNotFound = false)
{
if ($this->result) {
$nestedResult = new NestedResult($this->result);
if ($this->searchConfig->getResult()) {
$nestedResult = new NestedResult($this->searchConfig->getRows());
$root = $nestedResult->getRoot($this->data['nesting'], $this->data['index']);
$this->renderNode($root);
} elseif ($showNotFound) {

View File

@ -14,25 +14,18 @@ class AggregationTable extends Aggregation
/** @var array for summing up columns */
protected $sums;
/** @var string[] the result PIDs for each row */
protected $resultPIDs;
protected $resultRids;
protected $resultRevs;
public function __construct($id, $mode, \Doku_Renderer $renderer, SearchConfig $searchConfig)
{
parent::__construct($id, $mode, $renderer, $searchConfig);
$this->resultPIDs = $this->searchConfig->getPids();
$this->resultRids = $this->searchConfig->getRids();
$this->resultRevs = $this->searchConfig->getRevs();
}
/** @inheritdoc */
public function render($showNotFound = false)
{
if (in_array($this->mode, \helper_plugin_struct::BLACKLIST_RENDERER)) return;
// abort early if there are no results at all (not filtered)
if (!$this->resultCount && !$this->isDynamicallyFiltered() && $showNotFound) {
if ($this->searchConfig->getCount() <= 0 && !$this->isDynamicallyFiltered() && $showNotFound) {
$this->renderer->cdata($this->helper->getLang('none'));
return;
}
@ -45,7 +38,7 @@ class AggregationTable extends Aggregation
'format' => $this->mode,
'search' => $this->searchConfig,
'columns' => $this->columns,
'data' => $this->result
'data' => $this->searchConfig->getRows()
];
$event = new Event(
@ -71,7 +64,7 @@ class AggregationTable extends Aggregation
$this->renderDynamicFilters();
$this->renderer->tablethead_close();
if ($this->resultCount) {
if ($this->searchConfig->getCount()) {
// actual data
$this->renderer->tabletbody_open();
$this->renderResult();
@ -324,7 +317,7 @@ class AggregationTable extends Aggregation
*/
protected function renderResult()
{
foreach ($this->result as $rownum => $row) {
foreach ($this->searchConfig->getRows() as $rownum => $row) {
$data = [
'id' => $this->id,
'mode' => $this->mode,
@ -354,9 +347,9 @@ class AggregationTable extends Aggregation
// add data attribute for inline edit
if ($this->mode == 'xhtml') {
$pid = $this->resultPIDs[$rownum];
$rid = $this->resultRids[$rownum];
$rev = $this->resultRevs[$rownum];
$pid = $this->searchConfig->getPids()[$rownum];
$rid = $this->searchConfig->getRids()[$rownum];
$rev = $this->searchConfig->getRevs()[$rownum];
$this->renderer->doc = substr(rtrim($this->renderer->doc), 0, -1); // remove closing '>'
$this->renderer->doc .= ' data-pid="' . hsc($pid) . '" data-rev="' . $rev . '" data-rid="' . $rid . '">';
}
@ -407,7 +400,6 @@ class AggregationTable extends Aggregation
$this->renderer->info['struct_table_meta'] = true;
if ($this->mode == 'xhtml') {
/** @noinspection PhpMethodParametersCountMismatchInspection */
$this->renderer->tablerow_open('summarize');
} else {
$this->renderer->tablerow_open();
@ -463,7 +455,7 @@ class AggregationTable extends Aggregation
}
// next link
if ($this->resultCount > $offset + $limit) {
if ($this->searchConfig->getCount() > $offset + $limit) {
$next = $offset + $limit;
$dynamic = $this->searchConfig->getDynamicParameters();
$dynamic->setOffset($next);
@ -483,7 +475,7 @@ class AggregationTable extends Aggregation
{
if ($this->mode != 'xhtml') return;
if (empty($this->data['csv'])) return;
if (!$this->resultCount) return;
if (!$this->searchConfig->getCount()) return;
$dynamic = $this->searchConfig->getDynamicParameters();
$params = $dynamic->getURLParameters();

View File

@ -34,7 +34,7 @@ class CSVExporter
$search->addSchema($table);
$search->addColumn('*');
$result = $search->execute();
$result = $search->getRows();
if ($this->type !== self::DATATYPE_GLOBAL) {
$pids = $search->getPids();

View File

@ -2,6 +2,7 @@
namespace dokuwiki\plugin\struct\meta;
use dokuwiki\Debug\DebugHelper;
use dokuwiki\Parsing\Lexer\Lexer;
use dokuwiki\plugin\struct\types\AutoSummary;
use dokuwiki\plugin\struct\types\DateTime;
@ -52,14 +53,10 @@ class Search
/** @var int end results here */
protected $range_end = 0;
/** @var int the number of results */
protected $count = -1;
/** @var string[] the PIDs of the result rows */
protected $result_pids;
/** @var array the row ids of the result rows */
protected $result_rids = [];
/** @var array the revisions of the result rows */
protected $result_revs = [];
/**
* @var SearchResult
*/
protected $result;
/** @var bool Include latest = 1 in select query */
protected $selectLatest = true;
@ -384,59 +381,64 @@ class Search
$this->selectLatest = $selectLatest;
}
/**
* If the search result object does not exist yet,
* the search is run and the result object returned
*
* @return SearchResult
*/
public function getResult()
{
if (is_null($this->result)) {
$this->run();
}
return $this->result;
}
/**
* Return the number of results (regardless of limit and offset settings)
*
* Use this to implement paging. Important: this may only be called after running @return int
* @see execute()
*
*/
public function getCount()
{
if ($this->count < 0) throw new StructException('Count is only accessible after executing the search');
return $this->count;
return $this->getResult()->getCount();
}
/**
* Returns the PID associated with each result row
*
* Important: this may only be called after running @return \string[]
* @see execute()
*
*/
public function getPids()
{
if ($this->result_pids === null)
throw new StructException('PIDs are only accessible after executing the search');
return $this->result_pids;
return $this->getResult()->getPids();
}
/**
* Returns the rid associated with each result row
*
* Important: this may only be called after running @return array
* @see execute()
*
* @return array
*/
public function getRids()
{
if ($this->result_rids === null)
throw new StructException('rids are only accessible after executing the search');
return $this->result_rids;
return $this->getResult()->getRids();
}
/**
* Returns the rid associated with each result row
*
* Important: this may only be called after running @return array
* @see execute()
* Returns the revisions of search results
*
* @return array
*/
public function getRevs()
{
if ($this->result_revs === null)
throw new StructException('revs are only accessible after executing the search');
return $this->result_revs;
return $this->getResult()->getRevs();
}
/**
* Returns the actual result rows
*
* @return Value[][]
*/
public function getRows()
{
return $this->getResult()->getRows();
}
/**
@ -445,11 +447,23 @@ class Search
* The result is a two dimensional array of Value()s.
*
* This will always query for the full result (not using offset and limit) and then
* return the wanted range, setting the count (@return Value[][]
* @see getCount) to the whole result number
* return the wanted range, setting the count to the whole result number
*
* @deprecated Use getRows() instead
* @return Value[][]
*/
public function execute()
{
DebugHelper::dbgDeprecatedFunction('\dokuwiki\plugin\struct\meta\Search::getRows()');
return $this->getRows();
}
/**
* Run the actual search and populate the result object
*
* @return void
*/
protected function run()
{
[$sql, $opts] = $this->getSQL();
@ -457,48 +471,14 @@ class Search
$res = $this->sqlite->query($sql, $opts);
if ($res === false) throw new StructException("SQL execution failed for\n\n$sql");
$this->result_pids = [];
$result = [];
$cursor = -1;
$pageidAndRevOnly = array_reduce(
$this->columns,
static fn($pageidAndRevOnly, Column $col) => $pageidAndRevOnly && ($col->getTid() == 0),
true
);
while ($row = $res->fetch(\PDO::FETCH_ASSOC)) {
$cursor++;
if ($cursor < $this->range_begin) continue;
if ($this->range_end && $cursor >= $this->range_end) continue;
$C = 0;
$resrow = [];
$isempty = true;
foreach ($this->columns as $col) {
$val = $row["C$C"];
if ($col->isMulti()) {
$val = explode(self::CONCAT_SEPARATOR, $val);
}
$value = new Value($col, $val);
$isempty &= $this->isEmptyValue($value);
$resrow[] = $value;
$C++;
}
// skip empty rows
if ($isempty && !$pageidAndRevOnly) {
$cursor--;
continue;
}
$this->result_pids[] = $row['PID'];
$this->result_rids[] = $row['rid'];
$this->result_revs[] = $row['rev'];
$result[] = $resrow;
}
$this->result = new SearchResult($res, $this->range_begin, $this->range_end, $this->columns, $pageidAndRevOnly);
$res->closeCursor();
$this->count = $cursor + 1;
return $result;
}
/**
@ -667,17 +647,4 @@ class Search
return $col;
}
/**
* Check if the given row is empty or references our own row
*
* @param Value $value
* @return bool
*/
protected function isEmptyValue(Value $value)
{
if ($value->isEmpty()) return true;
if ($value->getColumn()->getTid() == 0) return true;
return false;
}
}

View File

@ -123,7 +123,6 @@ class SearchCloud extends SearchConfig
}
$res->closeCursor();
$this->count = count($result);
return $result;
}
}

134
meta/SearchResult.php Normal file
View File

@ -0,0 +1,134 @@
<?php
namespace dokuwiki\plugin\struct\meta;
/**
* Class SearchResult
*
* Search is executed only once per request.
*/
class SearchResult
{
/** @var Value[][] */
protected $rows = [];
/** @var array */
protected $pids = [];
protected $rids = [];
/** @var array */
protected $revs = [];
/** @var int */
protected $count = -1;
/**
* Construct SearchResult
*
* @param \PDOStatement $res PDO statement containing the search result
* @param int $rangeBegin Begin of requested result range
* @param int $rangeEnd End of requested result range
* @param Column[] $columns Search columns
* @param bool $pageidAndRevOnly
*/
public function __construct($res, $rangeBegin, $rangeEnd, $columns, $pageidAndRevOnly)
{
while ($row = $res->fetch(\PDO::FETCH_ASSOC)) {
$this->increaseCount();
if ($this->getCount() < $rangeBegin) continue;
if ($rangeEnd && $this->getCount() >= $rangeEnd) continue;
$C = 0;
$resrow = [];
$isempty = true;
foreach ($columns as $col) {
$val = $row["C$C"];
if ($col->isMulti()) {
$val = explode(Search::CONCAT_SEPARATOR, $val);
}
$value = new Value($col, $val);
$isempty &= $this->isEmptyValue($value);
$resrow[] = $value;
$C++;
}
// skip empty rows
if ($isempty && !$pageidAndRevOnly) {
$this->decreaseCount();
continue;
}
$this->pids[] = $row['PID'];
$this->rids[] = $row['rid'];
$this->revs[] = $row['rev'];
$this->rows[] = $resrow;
}
$this->increaseCount();
}
/**
* @return array
*/
public function getPids(): array
{
return $this->pids;
}
/**
* @return Value[][]
*/
public function getRows()
{
return $this->rows;
}
/**
* @return array
*/
public function getRids(): array
{
return $this->rids;
}
/**
* @return int
*/
public function getCount(): int
{
return $this->count;
}
/**
* @return array
*/
public function getRevs(): array
{
return $this->revs;
}
/**
* @return void
*/
public function increaseCount()
{
$this->count++;
}
/**
* @return void
*/
public function decreaseCount()
{
$this->count--;
}
/**
* Check if the given row is empty or references our own row
*
* @param Value $value
* @return bool
*/
protected function isEmptyValue(Value $value)
{
if ($value->isEmpty()) return true;
if ($value->getColumn()->getTid() == 0) return true;
return false;
}
}

View File

@ -152,7 +152,7 @@ class remote_plugin_struct extends RemotePlugin
$parser = new ConfigParser(array_merge([$schemaLine, $columnLine, $sortLine], $filterLines));
$config = $parser->getConfig();
$search = new SearchConfig($config);
$results = $search->execute();
$results = $search->getRows();
$data = [];
/** @var Value[] $rowValues */
foreach ($results as $rowValues) {

View File

@ -124,7 +124,7 @@ class Lookup extends Dropdown
$search->addColumn($field);
$search->addSort($field);
$result = $search->execute();
$result = $search->getRows();
$pids = $search->getPids();
$rids = $search->getRids();
$len = count($result);