Matt Housh
e78df72edd
* rss generator: minor updates for XML output formatting and PHP log warnings * cacher: replaced flyspray task caching with gitea issues caching
386 lines
16 KiB
PHP
Executable File
386 lines
16 KiB
PHP
Executable File
#!/usr/bin/php
|
|
<?php
|
|
|
|
/*
|
|
Caches timeline events from git, gitea issues, and pmwiki
|
|
into a SQLite3 database.
|
|
*/
|
|
|
|
|
|
/* configuration */
|
|
require_once('tlcacher_config.php');
|
|
|
|
// server timezone
|
|
$tz = new DateTimeZone("Europe/Stockholm");
|
|
|
|
// gitea base URL
|
|
$base_url = "https://git.crux.nu";
|
|
|
|
// gitea commit base URL
|
|
$git_url = "{$base_url}/%s/commit/%s";
|
|
|
|
|
|
// git repos for which to cache events
|
|
// append ":branch" to the repo name to restrict logs to 'branch'
|
|
$git_repos = array("ports/core:3.7", "ports/opt:3.7", "ports/xorg:3.7", "ports/compat-32:3.7", "tools/pkgutils", "system/iso");
|
|
|
|
// path to git repositories on disk
|
|
$git_root = "/home/gitea/git";
|
|
|
|
// Map git authors to wiki profiles
|
|
$git_username_map = array(
|
|
"Per Lidén" => "PerLiden",
|
|
"Matt Housh" => "MattHoush",
|
|
"Juergen Daubert" => "JuergenDaubert",
|
|
"Johannes Winkelmann" => "JohannesWinkelmann",
|
|
"Simone Rota" => "SimoneRota" ,
|
|
"Jason Thomas Dolan" => "JasonThomasDolan",
|
|
"Jukka Heino" => "JukkaHeino",
|
|
"Tilman Sauerbeck" => "TilmanSauerbeck",
|
|
"Simon Gloßner" => "SimonGloßner",
|
|
"Nick Steeves" => "NickSteeves",
|
|
"Antti Nykänen" => "AnttiNykänen",
|
|
"Antti Nykanen" => "AnttiNykänen",
|
|
"Jose V Beneyto" => "JoseVBeneyto",
|
|
"JoseVBeneyto" => "JoseVBeneyto",
|
|
"Lucas Hazel" => "LucasHazel",
|
|
"Thomas Penteker" => "ThomasPenteker",
|
|
"Fredrik Rinnestam" => "FredrikRinnestam",
|
|
"Danny Rawlins" => "DannyRawlins",
|
|
"Tim Biermann" => "TimBiermann",
|
|
);
|
|
|
|
// Path of the recent changes pmwiki file
|
|
$wiki_file = '/var/www/pmwiki/wiki.d/Site.AllRecentChanges';
|
|
|
|
// Event: cache_id, tstamp, type(icon), date, time, user, url, description, notes
|
|
$events = array();
|
|
|
|
/**************** Last cached events *******************/
|
|
|
|
$dbc = new SQLite3($dbfilec);
|
|
|
|
print("Fetching last cached gitea issues event.");
|
|
$last_cached_sql = "SELECT cache_id FROM events WHERE event_type LIKE 'task%' ORDER BY cache_id DESC LIMIT 1";
|
|
$res = $dbc->query($last_cached_sql);
|
|
if (!$res) die ("Query error: $last_cached_sql");
|
|
$row = $res->fetchArray(SQLITE3_ASSOC);
|
|
if (!$row) {
|
|
$last_task = 0;
|
|
} else {
|
|
$last_task = $row['cache_id'];
|
|
}
|
|
print(" [$last_task]\n");
|
|
|
|
print("Fetching last cached wiki event.");
|
|
$last_cached_sql = "select cache_id from events where event_type = 'wiki_changed' order by cache_id desc limit 1";
|
|
$res = $dbc->query($last_cached_sql);
|
|
if (!$res) die ("Query error: $last_cached_sql");
|
|
$row = $res->fetchArray(SQLITE3_ASSOC);
|
|
if (!$row) {
|
|
$last_wiki = 0;
|
|
} else {
|
|
$last_wiki = $row['cache_id'];
|
|
}
|
|
print(" [$last_wiki]\n");
|
|
|
|
function last_cached_git($repo) {
|
|
global $dbc;
|
|
$last_cached_sql = "select cache_id from events where event_type='git_commit_$repo' order by event_tstamp desc limit 1";
|
|
$res = $dbc->query($last_cached_sql);
|
|
if (!$res) die ("Query error: $last_cached_sql");
|
|
$row = $res->fetchArray(SQLITE3_ASSOC);
|
|
if (!$row) {
|
|
$last_git = "";
|
|
} else {
|
|
$last_git = $row['cache_id'];
|
|
}
|
|
return $last_git;
|
|
}
|
|
|
|
|
|
/* gitea issues */
|
|
|
|
print("\nProcessing gitea issues events.\n");
|
|
|
|
$pgdbc = pg_connect("host={$dbhost} dbname={$dbname} user={$dbuser} password={$dbpass}")
|
|
or die ("Failed to connect to database: ". pg_last_error());
|
|
|
|
// get issues
|
|
$query = 'SELECT i.id, i.repo_id, r.owner_name, r.name AS repo_name, i.index, i.is_closed, i.name, u.full_name, i.created_unix, i.updated_unix, i.closed_unix FROM issue AS i JOIN "user" AS u ON i.poster_id = u.id JOIN repository AS r ON r.id = i.repo_id WHERE r.is_private = false';
|
|
$result = pg_query($query)
|
|
or die ("Failed to execute issues query: " . pg_last_error());
|
|
|
|
$events = array();
|
|
while ($row = pg_fetch_array($result, null, PGSQL_ASSOC)) {
|
|
|
|
// construct date/time from unix timestamp
|
|
$date_im = date_create_immutable_from_format("U", $row['created_unix'], $tz)->setTimezone($tz);
|
|
$date = date_format($date_im, 'Y-m-d');
|
|
$time = date_format($date_im, 'H:i');
|
|
|
|
// construct issue URL
|
|
$issue_url = $base_url . "/" . $row['owner_name'] . "/" . $row['repo_name'] . "/issues/" . $row['index'];
|
|
|
|
// construct wiki user
|
|
$user = $row['full_name'];
|
|
$userlink = $user;
|
|
if (array_key_exists($user, $git_username_map)) {
|
|
$wikiname = $git_username_map[$user];
|
|
$userlink = "[[~{$wikiname}|{$user}]]";
|
|
}
|
|
|
|
// construct the array wanted by the cacher database schema
|
|
$events[] = array(
|
|
"cache_id" => $row['created_unix'],
|
|
"tstamp" => $row['created_unix'],
|
|
"icon" => "task_opened",
|
|
"date" => $date,
|
|
"time" => $time,
|
|
"user" => $user,
|
|
"url" => $issue_url,
|
|
"description" => "New issue [[ {$issue_url} | {$row['owner_name']}/{$row['repo_name']} #{$row['index']} ]] opened by {$userlink}",
|
|
"notes" => $row['name']
|
|
);
|
|
}
|
|
|
|
|
|
// get comments/events (gitea combines them)
|
|
$query = 'SELECT c.id, c.issue_id, r.owner_name, r.name AS repo_name, i.index, c.type, c.poster_id, u.full_name, c.created_unix, c.updated_unix, (select c.updated_unix = c.created_unix) AS orig, c.content, c.commit_sha FROM comment AS c LEFT JOIN "user" AS u ON u.id = c.poster_id LEFT JOIN issue AS i ON i.id = c.issue_id LEFT JOIN repository AS r ON r.id = i.repo_id WHERE r.is_private = false';
|
|
$result = pg_query($query)
|
|
or die ("Failed to execute comments query: " . pg_last_error());
|
|
|
|
//$comments = array();
|
|
while ($row = pg_fetch_array($result, null, PGSQL_ASSOC)) {
|
|
|
|
// construct date/time from unix timestamp
|
|
$date_im = date_create_immutable_from_format("U", $row['created_unix'], $tz)->setTimezone($tz);
|
|
$date = date_format($date_im, 'Y-m-d');
|
|
$time = date_format($date_im, 'H:i');
|
|
|
|
// construct issue URL
|
|
$issue_url = $base_url . '/' . $row['owner_name'] . '/' . $row['repo_name'] . '/issues/' . $row['index'];
|
|
|
|
// construct wiki user
|
|
$user = $row['full_name'];
|
|
$userlink = $user;
|
|
if (array_key_exists($user, $git_username_map)) {
|
|
$wikiname = $git_username_map[$user];
|
|
$userlink = "[[~{$wikiname}|{$user}]]";
|
|
}
|
|
|
|
// determine the message (description) by the comment type (https://github.com/go-gitea/gitea/blob/main/models/issues/comment.go#L60)
|
|
$desc = '';
|
|
switch ($row['type']) {
|
|
case 0: // comment
|
|
$desc = "{$userlink} commented on issue [[ {$issue_url} | {$row['owner_name']}/{$row['repo_name']} #{$row['index']} ]]";
|
|
break;
|
|
case 4: // referenced by commit
|
|
$shortsha = substr($row['commit_sha'], 0, 7);
|
|
$desc = "{$userlink} referenced issue [[ {$issue_url} | {$row['owner_name']}/{$row['repo_name']} #{$row['index']} ]] in commit [[ {$base_url}/{$row['owner_name']}/{$row['repo_name']}/commit/{$row['commit_sha']} | {$shortsha} ]]";
|
|
break;
|
|
case 7: // labels changed
|
|
$desc = "{$userlink} changed label(s) on issue [[ {$issue_url} | {$row['owner_name']}/{$row['repo_name']} #{$row['index']} ]]";
|
|
break;
|
|
case 9: // assignees changed
|
|
$desc = "{$userlink} changed assignee(s) on issue [[ {$issue_url} | {$row['owner_name']}/{$row['repo_name']} #{$row['index']} ]]";
|
|
break;
|
|
case 28: // PR merge
|
|
$desc = "{$userlink} merged a pull request for issue [[ {$issue_url} | {$row['owner_name']}/{$row['repo_name']} #{$row['index']} ]]";
|
|
break;
|
|
case 2: // close comment
|
|
case 29: // PR head branch push
|
|
default:
|
|
$desc = "{$userlink} modified issue [[ {$issue_url} | {$row['owner_name']}/{$row['repo_name']} #{$row['index']} ]]";
|
|
break;
|
|
}
|
|
|
|
// construct the array wanted by the cacher database schema
|
|
$events[] = array(
|
|
'cache_id' => $row['created_unix'],
|
|
'tstamp' => $row['created_unix'],
|
|
'icon' => 'task_changed',
|
|
'date' => $date,
|
|
'time' => $time,
|
|
'user' => $user,
|
|
'url' => $issue_url,
|
|
'description' => $desc,
|
|
'notes' => ''
|
|
);
|
|
}
|
|
|
|
|
|
// get closures (this is just a list of all issues with status closed in non-private repos)
|
|
$query = 'SELECT c.id, c.type, c.issue_id, i.repo_id, i.index, r.owner_name, r.name as repo_name, u.full_name, i.closed_unix FROM comment AS c JOIN issue AS i ON c.issue_id = i.id JOIN repository AS r ON r.id = i.repo_id JOIN "user" AS u ON c.poster_id = u.id WHERE c.type = 2 AND r.is_private = false';
|
|
$result = pg_query($query)
|
|
or die ("Failed to execute closures query: " . pg_last_Error());
|
|
|
|
while ($row = pg_fetch_array($result, null, PGSQL_ASSOC)) {
|
|
|
|
// construct date/time from unix timestamp
|
|
$date_im = date_create_immutable_from_format("U", $row['closed_unix'], $tz)->setTimezone($tz);
|
|
$date = date_format($date_im, 'Y-m-d');
|
|
$time = date_format($date_im, 'H:i');
|
|
|
|
// construct issue URL
|
|
$issue_url = $base_url . '/' . $row['owner_name'] . '/' . $row['repo_name'] . '/issues/' . $row['index'];
|
|
|
|
// construct wiki user
|
|
$user = $row['full_name'];
|
|
$userlink = $user;
|
|
if (array_key_exists($user, $git_username_map)) {
|
|
$wikiname = $git_username_map[$user];
|
|
$userlink = "[[~{$wikiname}|{$user}]]";
|
|
}
|
|
|
|
// construct the array wanted by the cacher database schema
|
|
$events[] = array(
|
|
'cache_id' => $row['closed_unix'],
|
|
'tstamp' => $row['closed_unix'],
|
|
'icon' => 'task_closed',
|
|
'date' => $date,
|
|
'time' => $time,
|
|
'user' => $user,
|
|
'url' => $issue_url,
|
|
"description" => "{$userlink} closed issue [[ {$issue_url} | {$row['owner_name']}/{$row['repo_name']} #{$row['index']} ]]",
|
|
'notes' => ''
|
|
);
|
|
}
|
|
|
|
|
|
// cleanup
|
|
pg_free_result($result);
|
|
pg_close($pgdbc);
|
|
|
|
|
|
/****************** PmWiki events *********************/
|
|
|
|
print("\nProcessing wiki events.\n");
|
|
|
|
$lines = file($wiki_file);
|
|
$chline = "";
|
|
foreach ($lines as $line) {
|
|
$line = urldecode($line);
|
|
if (substr($line,0,5) == "text=") {
|
|
$chline = substr($line,7);
|
|
}
|
|
}
|
|
|
|
if ($chline != "") {
|
|
$wikiedits = explode("*", $chline);
|
|
$icon = "wiki_changed";
|
|
foreach ($wikiedits as $ed) {
|
|
preg_match('/\[\[.*\]\] ./', $ed, $matches);
|
|
$page = $matches[0];
|
|
preg_match('/by \[\[.*\]\]/', $ed, $matches);
|
|
if (sizeof($matches) != 0) {
|
|
$user = $matches[0];
|
|
} else {
|
|
$user = "unknown";
|
|
}
|
|
preg_match("/\=(.*)\=/s",$ed,$matches);
|
|
$notes = $matches[1];
|
|
preg_match("/\. \. \. (.*?) by/s",$ed,$matches);
|
|
$date = $matches[1];
|
|
$date = str_replace(", at", "", $date); // old entry format
|
|
$tstamp = strtotime($date);
|
|
preg_match('/(..\:..)/',$date,$matches);
|
|
$time = $matches[0];
|
|
$date = date("Y-m-d", $tstamp);
|
|
$action = "?action=diff#" . urlencode($date . " " . $time) . "|diff";
|
|
$page_diff = trim(str_replace("]]", $action."]]", $page));
|
|
preg_match('/\[\[(.*)\|diff\]\]/', $page_diff, $matches);
|
|
$url = $matches[1];
|
|
$url = "https://crux.nu/".str_replace(".","/", $url);
|
|
$description = "Wiki page $page edited $user ($page_diff)";
|
|
if ($tstamp > $last_wiki) {
|
|
$events[] = array( 'cache_id' => $tstamp, 'tstamp' => $tstamp, 'icon' => $icon, 'date' => $date,
|
|
'time' => $time, 'user' => $user, 'url' => $url, 'description' => $description, 'notes' => $notes);
|
|
}
|
|
//print_r($events);
|
|
}
|
|
}
|
|
|
|
/******************* Git events ***********************/
|
|
|
|
print("\nProcessing git events.\n");
|
|
|
|
foreach ($git_repos as $repo) {
|
|
$branch = "";
|
|
if (strpos($repo, ":") !== FALSE) {
|
|
$tmp = explode(':', $repo);
|
|
$repo = $tmp[0];
|
|
$branch = $tmp[1];
|
|
}
|
|
$pos = strpos($repo, "/");
|
|
$reponame = substr($repo, $pos+1);
|
|
$last_git = last_cached_git($reponame);
|
|
print("Last cached commit for $repo: $last_git\n");
|
|
$out = array();
|
|
$res = 0;
|
|
$done = array();
|
|
exec("GIT_DIR=$git_root/$repo.git git log -n 1 $branch", $out, $res);
|
|
$git_latest = trim(str_replace("commit ", "", $out[0]));
|
|
unset($out);
|
|
$res = 0;
|
|
if ($git_latest != $last_git) {
|
|
if ($last_git == "") {
|
|
exec("GIT_DIR=$git_root/$repo.git git log $branch", $out, $res);
|
|
} else {
|
|
exec("GIT_DIR=$git_root/$repo.git git log $last_git..$git_latest $branch", $out, $res);
|
|
}
|
|
$last_git = $git_latest;
|
|
foreach ($out as $line) {
|
|
if (substr($line, 0, 7) == "commit ") {
|
|
$rev = substr($line, 7);
|
|
$url = sprintf($git_url, $repo, $rev);
|
|
$compact_rev = substr($rev,0,4)."..".substr($rev,-4,4);
|
|
$revurl = "[[$url|$compact_rev]]";
|
|
} else if (substr($line, 0, 7) == "Merge: ") {
|
|
} else if (substr($line, 0, 8) == "Author: ") {
|
|
$user = substr($line, 8);
|
|
$pos = strpos($user, "<");
|
|
if ($pos !== FALSE)
|
|
$user = trim(substr($user,0,$pos));
|
|
if (array_key_exists($user, $git_username_map)) {
|
|
$wikiname = $git_username_map[$user];
|
|
$user = "[[~" . $wikiname . "|" . $user . "]]";
|
|
}
|
|
$description = "$revurl committed by $user";
|
|
} else if (substr($line, 0, 8) == "Date: ") {
|
|
$date = substr($line, 8);
|
|
$tstamp = strtotime($date);
|
|
$date = date("Y-m-d", $tstamp);
|
|
$time = date("H:i", $tstamp);
|
|
} else if (trim($line) != "" && !array_key_exists($rev, $done)) {
|
|
$icon = "git_commit_$reponame";
|
|
$notes = trim($line);
|
|
$events[] = array( 'cache_id' => $rev, 'tstamp' => $tstamp, 'icon' => $icon, 'date' => $date,
|
|
'time' => $time, 'user' => $user, 'url'=>$url, 'description' => $description, 'notes' => $notes);
|
|
$done[$rev] = 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/*************** Finally, all events *********************/
|
|
$sql = 'INSERT INTO events VALUES (NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?)';
|
|
foreach ($events as $evt) {
|
|
$stmt = $dbc->prepare($sql);
|
|
$stmt->bindValue(1, $evt['cache_id'], SQLITE3_TEXT);
|
|
$stmt->bindValue(2, $evt['tstamp'], SQLITE3_INTEGER);
|
|
$stmt->bindValue(3, $evt['icon'], SQLITE3_TEXT);
|
|
$stmt->bindValue(4, $evt['date'], SQLITE3_TEXT);
|
|
$stmt->bindValue(5, $evt['time'], SQLITE3_TEXT);
|
|
$stmt->bindValue(6, $evt['user'], SQLITE3_TEXT);
|
|
$stmt->bindValue(7, $evt['url'], SQLITE3_TEXT);
|
|
$stmt->bindValue(8, $evt['description'], SQLITE3_TEXT);
|
|
$stmt->bindValue(9, $evt['notes'], SQLITE3_TEXT);
|
|
$res = $stmt->execute();
|
|
if (!$res) die ("Query error: $sql"); // this is unfortunately not very useful
|
|
}
|
|
|
|
// vim: set ts=4 et:
|
|
|
|
?>
|