From b9094bfd90c5e635e08a7b3423fb7bb9d3d8ffa0 Mon Sep 17 00:00:00 2001 From: lacatoire Date: Tue, 4 Aug 2026 11:43:19 +0200 Subject: [PATCH] Add "Do not translate" and "XML Broken" sections to revcheck Both were already being computed and then dropped. Files marked were detected and skipped without a trace, and libxml errors were collected by XmlUtil then cleared without ever being read. They are now listed, in the file summary and in a section of their own. Telling a real error apart from an unresolved entity was already solved in scripts/broken.php. That logic moves to a shared XmlErrorFilter, so both callers follow the same rule. Entity files are skipped, as DTD fragments never parse as standalone XML. Files marked do not translate are listed but kept out of the totals, otherwise every translation would see its completion rate drop for files it is not expected to have. The files table gains an xmlError column, so the web report can say what is wrong, not only which file. An empty file raised a ValueError in XmlUtil::loadText() and aborted the run. It is now reported as any other broken file. --- scripts/broken.php | 41 +------ scripts/revcheck.php | 119 +++++++++++++++++-- scripts/translation/genrevdb.php | 14 ++- scripts/translation/lib/RevcheckData.php | 5 + scripts/translation/lib/RevcheckFileItem.php | 1 + scripts/translation/lib/RevcheckRun.php | 22 ++++ scripts/translation/lib/RevtagParser.php | 15 +++ scripts/translation/lib/XmlErrorFilter.php | 116 ++++++++++++++++++ scripts/translation/lib/XmlUtil.php | 11 +- scripts/translation/lib/all.php | 1 + 10 files changed, 293 insertions(+), 52 deletions(-) create mode 100644 scripts/translation/lib/XmlErrorFilter.php diff --git a/scripts/broken.php b/scripts/broken.php index bf93a017a7..b26d143c19 100644 --- a/scripts/broken.php +++ b/scripts/broken.php @@ -22,6 +22,8 @@ This tool also cares for directories marked with .xmlfragmentdir, so theses files are tested in relaxed semantics for XML Fragments. */ +require_once __DIR__ . '/translation/lib/XmlErrorFilter.php'; + ini_set( 'display_errors' , 1 ); ini_set( 'display_startup_errors' , 1 ); error_reporting( E_ALL ); @@ -69,32 +71,6 @@ function print_usage_exit( $cmd ) exit; } -function setup( string & $prefix , string & $suffix , string & $extra ) -{ - // Undefined entities generate TWO different error messages on libxml - // - "Entity '?' not defined" (for entity inside elements) - // - "Extra content at the end of the document" (entity outside elements) - - $inside = "&ZZZ;"; - $outside = "&ZZZ;"; - - $doc = new DOMDocument(); - $doc->recover = true; - $doc->resolveExternals = false; - $doc->substituteEntities = false; - libxml_use_internal_errors( true ); - - $doc->loadXML( $inside ); - $message = trim( libxml_get_errors()[0]->message ); - $message = str_replace( "ZZZ" , "\f" , $message ); - [ $prefix , $suffix ] = explode( "\f" , $message ); - libxml_clear_errors(); - - $doc->loadXML( $outside ); - $extra = trim( libxml_get_errors()[0]->message ); - libxml_clear_errors(); -} - function testFile( string $filename , bool $checkDnt , bool $fragmentDir = false ) { $contents = file_get_contents( $filename ); @@ -125,13 +101,8 @@ function testFile( string $filename , bool $checkDnt , bool $fragmentDir = false echo " Issue: Manual build may fail.\n"; echo " Path: $filename\n"; echo "\n"; - autofix_dos2unix( $filename ); } - static $prefix = "", $suffix = "", $extra = ""; - if ( $extra == "" ) - setup( $prefix , $suffix , $extra ); - $doc = new DOMDocument(); $doc->recover = true; $doc->resolveExternals = false; @@ -150,11 +121,11 @@ function testFile( string $filename , bool $checkDnt , bool $fragmentDir = false $message = trim( $error->message ); $hintFragDir = false; - if ( str_starts_with( $message , $prefix ) && str_ends_with( $message , $suffix ) ) + if ( XmlErrorFilter::isUndefinedEntity( $message ) ) continue; - //if ( $message == $extra ) // Disabled as unnecessary. Also, this indicates that some - // continue; // some entity reference is used at an unusual position. - if ( $message == $extra ) + // Extra content is not skipped, as it indicates that some entity + // reference is used at an unusual position. + if ( XmlErrorFilter::isExtraContent( $message ) ) $hintFragDir = true; $lin = $error->line; diff --git a/scripts/revcheck.php b/scripts/revcheck.php index f2e1055eb3..08af9afb4f 100644 --- a/scripts/revcheck.php +++ b/scripts/revcheck.php @@ -63,6 +63,8 @@ function print_html_all( RevcheckData $data ) print_html_notinen( $data ); print_html_revtag( $data ); print_html_untranslated( $data ); + print_html_donottranslate( $data ); + print_html_xmlbroken( $data ); print_html_footer(); } @@ -113,6 +115,8 @@ function print_html_menu( string $href ) | Not in EN tree | Missing or invalid revtag | Untranslated files +| Do not translate +| Broken XML

HTML; } @@ -178,24 +182,24 @@ function print_html_translators( RevcheckData $data ) HTML; + // Files that must not be translated are listed, but kept out of the + // totals: counting them would lower the completion rate of every + // translation, for files no translation is expected to ever have. + $filesTotal = 0; - foreach ( $data->fileSummary as $count ) - $filesTotal += $count; + foreach ( $data->fileSummary as $status => $count ) + if ( $status != RevcheckStatus::DoNotTranslate->value ) + $filesTotal += $count; + + $labels = $data->getSummaryLabels(); foreach( RevcheckStatus::cases() as $key ) { - $label = ""; + $label = $labels[ $key->value ] ?? ""; $count = $data->fileSummary[ $key->value ]; - $perc = number_format( $count / $filesTotal * 100 , 2 ) . "%"; - switch( $key ) - { - case RevcheckStatus::TranslatedOk: $label = "Up to date files"; break; - case RevcheckStatus::TranslatedOld: $label = "Outdated files"; break; - case RevcheckStatus::TranslatedWip: $label = "Work in progress"; break; - case RevcheckStatus::RevTagProblem: $label = "Revision tag missing/problem"; break; - case RevcheckStatus::NotInEnTree: $label = "Not in EN tree"; break; - case RevcheckStatus::Untranslated: $label = "Available for translation"; break; - } + $perc = $key == RevcheckStatus::DoNotTranslate || $filesTotal == 0 + ? "n/a" + : number_format( $count / $filesTotal * 100 , 2 ) . "%"; print << @@ -428,6 +432,95 @@ function print_html_untranslated( RevcheckData $data ) print "\n\n"; } +function print_html_donottranslate( RevcheckData $data ) +{ + print_html_menu("donottranslate"); + if ( $data->fileSummary[ RevcheckStatus::DoNotTranslate->value ] == 0 ) + { + echo "

No source file is marked do not translate.

\n\n"; + return; + } + + print << + + Files marked do not translate + kb + +HTML; + + $path = null; + foreach ( $data->fileDetail as $file ) + { + if ( $file->status != RevcheckStatus::DoNotTranslate ) + continue; + + if ( $path !== $file->path ) + { + $path = $file->path; + $header = $path == '' ? '/' : $path; + print " $header"; + } + + $name = $file->name; + $size = round( $file->size / 1024 ); + + print << + $name + $size + +HTML; + } + print "\n\n"; +} + +function print_html_xmlbroken( RevcheckData $data ) +{ + print_html_menu("xmlbroken"); + if ( $data->fileSummary[ RevcheckStatus::XmlBroken->value ] == 0 ) + { + echo "

Good, all translated files are valid XML.

\n\n"; + return; + } + + print << + + Broken XML files + Error + kb + +HTML; + + $path = null; + foreach ( $data->fileDetail as $file ) + { + if ( $file->status != RevcheckStatus::XmlBroken ) + continue; + + if ( $path !== $file->path ) + { + $path = $file->path; + $header = $path == '' ? '/' : $path; + print " $header"; + } + + $name = $file->name; + $size = round( $file->size / 1024 ); + $error = htmlspecialchars( $file->xmlError ); + + print << + $name + $error + $size + +HTML; + } + print "\n\n"; +} + function print_html_footer() { print_html_menu(""); diff --git a/scripts/translation/genrevdb.php b/scripts/translation/genrevdb.php index b2e4a3e4b7..ac2a76dba0 100644 --- a/scripts/translation/genrevdb.php +++ b/scripts/translation/genrevdb.php @@ -93,18 +93,25 @@ function generate( SQLite3 $db , string $lang ) , $file->hashLast , $file->hashDiff , $file->hashRvtg + , $file->xmlError ); + // Same total as scripts/revcheck.php: files that must not be + // translated are listed, but never counted against a translation. + $filesTotal = 0; - foreach( $data->fileSummary as $count ) - $filesTotal += $count; + foreach( $data->fileSummary as $status => $count ) + if ( $status != RevcheckStatus::DoNotTranslate->value ) + $filesTotal += $count; $labels = $data->getSummaryLabels(); foreach( $data->fileSummary as $status => $count ) db_insert( $db , "summary", $data->lang , $status , $labels[ $status ] , $count - , number_format( $count / $filesTotal * 100 , 2 ) . "%" + , $status == RevcheckStatus::DoNotTranslate->value || $filesTotal == 0 + ? "n/a" + : number_format( $count / $filesTotal * 100 , 2 ) . "%" ); $db->exec( 'COMMIT TRANSACTION' ); @@ -204,6 +211,7 @@ function db_create( $path ) : SQLite3 hashLast TEXT, hashDiff TEXT, hashRvtg TEXT, + xmlError TEXT, UNIQUE ( lang , path , name ) ); SQL; diff --git a/scripts/translation/lib/RevcheckData.php b/scripts/translation/lib/RevcheckData.php index 670faf7780..7761ddbb99 100644 --- a/scripts/translation/lib/RevcheckData.php +++ b/scripts/translation/lib/RevcheckData.php @@ -25,6 +25,8 @@ enum RevcheckStatus : string case RevTagProblem = 'RevTagProblem'; case NotInEnTree = 'NotInEnTree'; case Untranslated = 'Untranslated'; + case DoNotTranslate = 'DoNotTranslate'; + case XmlBroken = 'XmlBroken'; } class RevcheckData @@ -68,6 +70,8 @@ public function getSummaryLabels() : array $ret[ RevcheckStatus::RevTagProblem->value ] = "Revision tag missing/problem"; $ret[ RevcheckStatus::NotInEnTree->value ] = "Not in EN tree"; $ret[ RevcheckStatus::Untranslated->value ] = "Available for translation"; + $ret[ RevcheckStatus::DoNotTranslate->value ] = "Marked do not translate"; + $ret[ RevcheckStatus::XmlBroken->value ] = "Broken XML files"; return $ret; } } @@ -100,4 +104,5 @@ class RevcheckDataFile public string $hashLast; // The most recent commit hash, skipped or not public string $hashDiff; // The most recent, non [skip-revcheck] commit hash public string $hashRvtg = ""; // Revtag hash, if any + public string $xmlError = ""; // First real XML error, if any } diff --git a/scripts/translation/lib/RevcheckFileItem.php b/scripts/translation/lib/RevcheckFileItem.php index 1e8541226c..1b6f6fef1a 100644 --- a/scripts/translation/lib/RevcheckFileItem.php +++ b/scripts/translation/lib/RevcheckFileItem.php @@ -29,6 +29,7 @@ class RevcheckFileItem public RevcheckStatus $status; // target only public RevtagInfo|null $revtag; // target only + public string $xmlError = ""; // set on target by RevtagParser, copied to source by RevcheckRun private array $hashList; // source only private bool $hashStop; // source only diff --git a/scripts/translation/lib/RevcheckRun.php b/scripts/translation/lib/RevcheckRun.php index e9f82d354e..c5f150a4ba 100644 --- a/scripts/translation/lib/RevcheckRun.php +++ b/scripts/translation/lib/RevcheckRun.php @@ -33,6 +33,8 @@ class RevcheckRun public array $filesUntranslated = []; public array $filesNotInEn = []; public array $filesWip = []; + public array $filesDoNotTranslate = []; + public array $filesXmlBroken = []; public array $qaList = []; public RevcheckData $revData; @@ -83,7 +85,12 @@ private function calculateStatus() if ( $target == null ) { if ( RevcheckIgnore::byMark( "{$this->sourceDir}/{$source->file}" ) ) + { + $source->status = RevcheckStatus::DoNotTranslate; + $this->filesDoNotTranslate[] = $source; + $this->addData( $source , null ); continue; + } $source->status = RevcheckStatus::Untranslated; $this->filesUntranslated[] = $source; @@ -91,6 +98,20 @@ private function calculateStatus() continue; } + // XmlBroken + // + // Checked before the revtag, as a broken file makes every other + // check on it unreliable, revtag parsing included. + + if ( $target->xmlError != "" ) + { + $source->status = RevcheckStatus::XmlBroken; + $source->xmlError = $target->xmlError; + $this->filesXmlBroken[] = $source; + $this->addData( $source , $target->revtag ); + continue; + } + // RevTagProblem if ( $target->revtag == null || strlen( $target->revtag->revision ) != 40 ) @@ -168,6 +189,7 @@ private function addData( RevcheckFileItem $info , RevtagInfo|null $revtag = nul $file->status = $info->status; $file->hashLast = $info->hashLast; $file->hashDiff = $info->hashDiff; + $file->xmlError = $info->xmlError; $this->revData->addFile( $info->file , $file ); diff --git a/scripts/translation/lib/RevtagParser.php b/scripts/translation/lib/RevtagParser.php index 6ca79f1a19..ed99bc01d3 100644 --- a/scripts/translation/lib/RevtagParser.php +++ b/scripts/translation/lib/RevtagParser.php @@ -33,7 +33,22 @@ class RevtagParser static function parseDir( string $lang , RevcheckFileList $list ) { foreach( $list->iterator() as $entry ) + { $entry->revtag = RevtagParser::parseFile( $lang . '/' . $entry->file ); + + // Files are parsed here anyway, so reuse the errors already + // collected by XmlUtil, instead of loading everything again. + // + // Only .xml files are checked. Entity files are DTD fragments, + // never standalone XML, so they always fail to parse as such. + + if ( str_ends_with( $entry->file , '.xml' ) == false ) + continue; + + $error = XmlUtil::$lastErrors[0] ?? null; + if ( $error != null ) + $entry->xmlError = trim( $error->message ) . " [{$error->line},{$error->column}]"; + } } public static function parseFile( string $filename ): RevtagInfo|null diff --git a/scripts/translation/lib/XmlErrorFilter.php b/scripts/translation/lib/XmlErrorFilter.php new file mode 100644 index 0000000000..a34955a0b6 --- /dev/null +++ b/scripts/translation/lib/XmlErrorFilter.php @@ -0,0 +1,116 @@ + | + * +----------------------------------------------------------------------+ + * | Description: Tell apart real XML errors from undefined entities. | + * +----------------------------------------------------------------------+ + */ + +// No require of all.php here. This file is also used by scripts/broken.php, +// that is otherwise standalone, and does not need the revcheck library. + +class XmlErrorFilter +{ + private static bool $ready = false; + private static string $prefix = ""; + private static string $suffix = ""; + private static string $extra = ""; + + /** + * Manual files are parsed one by one, without any DTD, so every entity + * reference is reported as an error by libxml. Instead of hardcoding + * message texts, that change between libxml versions, provoke the two + * known messages and learn them at runtime. + * + * - "Entity '?' not defined" (entity inside elements) + * - "Extra content at the end of the document" (entity outside elements) + */ + private static function setup() : void + { + if ( XmlErrorFilter::$ready ) + return; + + $was = libxml_use_internal_errors( true ); + + $doc = new DOMDocument(); + $doc->recover = true; + $doc->resolveExternals = false; + $doc->substituteEntities = false; + + // Setup runs lazily, on first use, so the error buffer is cleared + // around the probes below: the probe errors must not leak into the + // next document parsed, and any pending error must not be read as + // a probe result. Callers always own a copy of their own errors + // before reaching this class, so nothing of value is dropped here. + + libxml_clear_errors(); + + $doc->loadXML( "&ZZZ;" ); + $message = trim( libxml_get_errors()[0]->message ); + $message = str_replace( "ZZZ" , "\f" , $message ); + [ XmlErrorFilter::$prefix , XmlErrorFilter::$suffix ] = explode( "\f" , $message ); + libxml_clear_errors(); + + $doc->loadXML( "&ZZZ;" ); + XmlErrorFilter::$extra = trim( libxml_get_errors()[0]->message ); + libxml_clear_errors(); + + libxml_use_internal_errors( $was ); + + XmlErrorFilter::$ready = true; + } + + /** An entity reference that no DTD was around to resolve. Expected, not an error. */ + public static function isUndefinedEntity( string $message ) : bool + { + XmlErrorFilter::setup(); + $message = trim( $message ); + return str_starts_with( $message , XmlErrorFilter::$prefix ) && + str_ends_with( $message , XmlErrorFilter::$suffix ); + } + + /** Usually an entity reference outside of any enclosing tag. Reported, with a hint. */ + public static function isExtraContent( string $message ) : bool + { + XmlErrorFilter::setup(); + return trim( $message ) == XmlErrorFilter::$extra; + } + + /** + * Drop undefined entity messages, keep everything else. + * + * @param LibXMLError[] $errors + * @return LibXMLError[] + */ + public static function filter( array $errors ) : array + { + if ( count( $errors ) == 0 ) // by far the most common case + return []; + + XmlErrorFilter::setup(); + + $prefix = XmlErrorFilter::$prefix; + $suffix = XmlErrorFilter::$suffix; + + $ret = []; + foreach( $errors as $error ) + { + $message = trim( $error->message ); + if ( str_starts_with( $message , $prefix ) && str_ends_with( $message , $suffix ) ) + continue; + $ret[] = $error; + } + return $ret; + } +} diff --git a/scripts/translation/lib/XmlUtil.php b/scripts/translation/lib/XmlUtil.php index 4173ec2d08..5eab7e26b0 100644 --- a/scripts/translation/lib/XmlUtil.php +++ b/scripts/translation/lib/XmlUtil.php @@ -21,6 +21,9 @@ class XmlUtil { + /** Real errors of the last loadText(), undefined entities already filtered out. */ + public static array $lastErrors = []; + public static function extractEntities( $filename ) { $was = libxml_use_internal_errors( true ); @@ -74,7 +77,13 @@ public static function loadText( $contents ):DOMDocument $doc->resolveExternals = false; $doc->substituteEntities = false; - $doc->loadXML( $contents ); + // An empty file is a ValueError, not a parse error, and would abort + // the whole run. Feed a blank instead, so libxml reports its own + // "Document is empty" and the file is listed as broken, like any other. + + $doc->loadXML( $contents == "" ? " " : $contents ); + + XmlUtil::$lastErrors = XmlErrorFilter::filter( libxml_get_errors() ); libxml_clear_errors(); libxml_use_internal_errors( $was ); diff --git a/scripts/translation/lib/all.php b/scripts/translation/lib/all.php index 843c046bbf..3a27605a27 100644 --- a/scripts/translation/lib/all.php +++ b/scripts/translation/lib/all.php @@ -35,4 +35,5 @@ require_once __DIR__ . '/RevcheckIgnore.php'; require_once __DIR__ . '/RevcheckRun.php'; require_once __DIR__ . '/RevtagParser.php'; +require_once __DIR__ . '/XmlErrorFilter.php'; require_once __DIR__ . '/XmlUtil.php';