diff --git a/scripts/broken.php b/scripts/broken.php index bf93a017a7..b26d143c19 100644 --- a/scripts/broken.php +++ b/scripts/broken.php @@ -22,6 +22,8 @@ This tool also cares for directories marked with .xmlfragmentdir, so theses files are tested in relaxed semantics for XML Fragments. */ +require_once __DIR__ . '/translation/lib/XmlErrorFilter.php'; + ini_set( 'display_errors' , 1 ); ini_set( 'display_startup_errors' , 1 ); error_reporting( E_ALL ); @@ -69,32 +71,6 @@ function print_usage_exit( $cmd ) exit; } -function setup( string & $prefix , string & $suffix , string & $extra ) -{ - // Undefined entities generate TWO different error messages on libxml - // - "Entity '?' not defined" (for entity inside elements) - // - "Extra content at the end of the document" (entity outside elements) - - $inside = "&ZZZ;"; - $outside = "&ZZZ;"; - - $doc = new DOMDocument(); - $doc->recover = true; - $doc->resolveExternals = false; - $doc->substituteEntities = false; - libxml_use_internal_errors( true ); - - $doc->loadXML( $inside ); - $message = trim( libxml_get_errors()[0]->message ); - $message = str_replace( "ZZZ" , "\f" , $message ); - [ $prefix , $suffix ] = explode( "\f" , $message ); - libxml_clear_errors(); - - $doc->loadXML( $outside ); - $extra = trim( libxml_get_errors()[0]->message ); - libxml_clear_errors(); -} - function testFile( string $filename , bool $checkDnt , bool $fragmentDir = false ) { $contents = file_get_contents( $filename ); @@ -125,13 +101,8 @@ function testFile( string $filename , bool $checkDnt , bool $fragmentDir = false echo " Issue: Manual build may fail.\n"; echo " Path: $filename\n"; echo "\n"; - autofix_dos2unix( $filename ); } - static $prefix = "", $suffix = "", $extra = ""; - if ( $extra == "" ) - setup( $prefix , $suffix , $extra ); - $doc = new DOMDocument(); $doc->recover = true; $doc->resolveExternals = false; @@ -150,11 +121,11 @@ function testFile( string $filename , bool $checkDnt , bool $fragmentDir = false $message = trim( $error->message ); $hintFragDir = false; - if ( str_starts_with( $message , $prefix ) && str_ends_with( $message , $suffix ) ) + if ( XmlErrorFilter::isUndefinedEntity( $message ) ) continue; - //if ( $message == $extra ) // Disabled as unnecessary. Also, this indicates that some - // continue; // some entity reference is used at an unusual position. - if ( $message == $extra ) + // Extra content is not skipped, as it indicates that some entity + // reference is used at an unusual position. + if ( XmlErrorFilter::isExtraContent( $message ) ) $hintFragDir = true; $lin = $error->line; diff --git a/scripts/revcheck.php b/scripts/revcheck.php index f2e1055eb3..08af9afb4f 100644 --- a/scripts/revcheck.php +++ b/scripts/revcheck.php @@ -63,6 +63,8 @@ function print_html_all( RevcheckData $data ) print_html_notinen( $data ); print_html_revtag( $data ); print_html_untranslated( $data ); + print_html_donottranslate( $data ); + print_html_xmlbroken( $data ); print_html_footer(); } @@ -113,6 +115,8 @@ function print_html_menu( string $href ) | Not in EN tree | Missing or invalid revtag | Untranslated files +| Do not translate +| Broken XML

HTML; } @@ -178,24 +182,24 @@ function print_html_translators( RevcheckData $data ) HTML; + // Files that must not be translated are listed, but kept out of the + // totals: counting them would lower the completion rate of every + // translation, for files no translation is expected to ever have. + $filesTotal = 0; - foreach ( $data->fileSummary as $count ) - $filesTotal += $count; + foreach ( $data->fileSummary as $status => $count ) + if ( $status != RevcheckStatus::DoNotTranslate->value ) + $filesTotal += $count; + + $labels = $data->getSummaryLabels(); foreach( RevcheckStatus::cases() as $key ) { - $label = ""; + $label = $labels[ $key->value ] ?? ""; $count = $data->fileSummary[ $key->value ]; - $perc = number_format( $count / $filesTotal * 100 , 2 ) . "%"; - switch( $key ) - { - case RevcheckStatus::TranslatedOk: $label = "Up to date files"; break; - case RevcheckStatus::TranslatedOld: $label = "Outdated files"; break; - case RevcheckStatus::TranslatedWip: $label = "Work in progress"; break; - case RevcheckStatus::RevTagProblem: $label = "Revision tag missing/problem"; break; - case RevcheckStatus::NotInEnTree: $label = "Not in EN tree"; break; - case RevcheckStatus::Untranslated: $label = "Available for translation"; break; - } + $perc = $key == RevcheckStatus::DoNotTranslate || $filesTotal == 0 + ? "n/a" + : number_format( $count / $filesTotal * 100 , 2 ) . "%"; print << @@ -428,6 +432,95 @@ function print_html_untranslated( RevcheckData $data ) print "\n\n"; } +function print_html_donottranslate( RevcheckData $data ) +{ + print_html_menu("donottranslate"); + if ( $data->fileSummary[ RevcheckStatus::DoNotTranslate->value ] == 0 ) + { + echo "

No source file is marked do not translate.

\n\n"; + return; + } + + print << + + Files marked do not translate + kb + +HTML; + + $path = null; + foreach ( $data->fileDetail as $file ) + { + if ( $file->status != RevcheckStatus::DoNotTranslate ) + continue; + + if ( $path !== $file->path ) + { + $path = $file->path; + $header = $path == '' ? '/' : $path; + print " $header"; + } + + $name = $file->name; + $size = round( $file->size / 1024 ); + + print << + $name + $size + +HTML; + } + print "\n\n"; +} + +function print_html_xmlbroken( RevcheckData $data ) +{ + print_html_menu("xmlbroken"); + if ( $data->fileSummary[ RevcheckStatus::XmlBroken->value ] == 0 ) + { + echo "

Good, all translated files are valid XML.

\n\n"; + return; + } + + print << + + Broken XML files + Error + kb + +HTML; + + $path = null; + foreach ( $data->fileDetail as $file ) + { + if ( $file->status != RevcheckStatus::XmlBroken ) + continue; + + if ( $path !== $file->path ) + { + $path = $file->path; + $header = $path == '' ? '/' : $path; + print " $header"; + } + + $name = $file->name; + $size = round( $file->size / 1024 ); + $error = htmlspecialchars( $file->xmlError ); + + print << + $name + $error + $size + +HTML; + } + print "\n\n"; +} + function print_html_footer() { print_html_menu(""); diff --git a/scripts/translation/genrevdb.php b/scripts/translation/genrevdb.php index b2e4a3e4b7..ac2a76dba0 100644 --- a/scripts/translation/genrevdb.php +++ b/scripts/translation/genrevdb.php @@ -93,18 +93,25 @@ function generate( SQLite3 $db , string $lang ) , $file->hashLast , $file->hashDiff , $file->hashRvtg + , $file->xmlError ); + // Same total as scripts/revcheck.php: files that must not be + // translated are listed, but never counted against a translation. + $filesTotal = 0; - foreach( $data->fileSummary as $count ) - $filesTotal += $count; + foreach( $data->fileSummary as $status => $count ) + if ( $status != RevcheckStatus::DoNotTranslate->value ) + $filesTotal += $count; $labels = $data->getSummaryLabels(); foreach( $data->fileSummary as $status => $count ) db_insert( $db , "summary", $data->lang , $status , $labels[ $status ] , $count - , number_format( $count / $filesTotal * 100 , 2 ) . "%" + , $status == RevcheckStatus::DoNotTranslate->value || $filesTotal == 0 + ? "n/a" + : number_format( $count / $filesTotal * 100 , 2 ) . "%" ); $db->exec( 'COMMIT TRANSACTION' ); @@ -204,6 +211,7 @@ function db_create( $path ) : SQLite3 hashLast TEXT, hashDiff TEXT, hashRvtg TEXT, + xmlError TEXT, UNIQUE ( lang , path , name ) ); SQL; diff --git a/scripts/translation/lib/RevcheckData.php b/scripts/translation/lib/RevcheckData.php index 670faf7780..7761ddbb99 100644 --- a/scripts/translation/lib/RevcheckData.php +++ b/scripts/translation/lib/RevcheckData.php @@ -25,6 +25,8 @@ enum RevcheckStatus : string case RevTagProblem = 'RevTagProblem'; case NotInEnTree = 'NotInEnTree'; case Untranslated = 'Untranslated'; + case DoNotTranslate = 'DoNotTranslate'; + case XmlBroken = 'XmlBroken'; } class RevcheckData @@ -68,6 +70,8 @@ public function getSummaryLabels() : array $ret[ RevcheckStatus::RevTagProblem->value ] = "Revision tag missing/problem"; $ret[ RevcheckStatus::NotInEnTree->value ] = "Not in EN tree"; $ret[ RevcheckStatus::Untranslated->value ] = "Available for translation"; + $ret[ RevcheckStatus::DoNotTranslate->value ] = "Marked do not translate"; + $ret[ RevcheckStatus::XmlBroken->value ] = "Broken XML files"; return $ret; } } @@ -100,4 +104,5 @@ class RevcheckDataFile public string $hashLast; // The most recent commit hash, skipped or not public string $hashDiff; // The most recent, non [skip-revcheck] commit hash public string $hashRvtg = ""; // Revtag hash, if any + public string $xmlError = ""; // First real XML error, if any } diff --git a/scripts/translation/lib/RevcheckFileItem.php b/scripts/translation/lib/RevcheckFileItem.php index 1e8541226c..1b6f6fef1a 100644 --- a/scripts/translation/lib/RevcheckFileItem.php +++ b/scripts/translation/lib/RevcheckFileItem.php @@ -29,6 +29,7 @@ class RevcheckFileItem public RevcheckStatus $status; // target only public RevtagInfo|null $revtag; // target only + public string $xmlError = ""; // set on target by RevtagParser, copied to source by RevcheckRun private array $hashList; // source only private bool $hashStop; // source only diff --git a/scripts/translation/lib/RevcheckRun.php b/scripts/translation/lib/RevcheckRun.php index e9f82d354e..c5f150a4ba 100644 --- a/scripts/translation/lib/RevcheckRun.php +++ b/scripts/translation/lib/RevcheckRun.php @@ -33,6 +33,8 @@ class RevcheckRun public array $filesUntranslated = []; public array $filesNotInEn = []; public array $filesWip = []; + public array $filesDoNotTranslate = []; + public array $filesXmlBroken = []; public array $qaList = []; public RevcheckData $revData; @@ -83,7 +85,12 @@ private function calculateStatus() if ( $target == null ) { if ( RevcheckIgnore::byMark( "{$this->sourceDir}/{$source->file}" ) ) + { + $source->status = RevcheckStatus::DoNotTranslate; + $this->filesDoNotTranslate[] = $source; + $this->addData( $source , null ); continue; + } $source->status = RevcheckStatus::Untranslated; $this->filesUntranslated[] = $source; @@ -91,6 +98,20 @@ private function calculateStatus() continue; } + // XmlBroken + // + // Checked before the revtag, as a broken file makes every other + // check on it unreliable, revtag parsing included. + + if ( $target->xmlError != "" ) + { + $source->status = RevcheckStatus::XmlBroken; + $source->xmlError = $target->xmlError; + $this->filesXmlBroken[] = $source; + $this->addData( $source , $target->revtag ); + continue; + } + // RevTagProblem if ( $target->revtag == null || strlen( $target->revtag->revision ) != 40 ) @@ -168,6 +189,7 @@ private function addData( RevcheckFileItem $info , RevtagInfo|null $revtag = nul $file->status = $info->status; $file->hashLast = $info->hashLast; $file->hashDiff = $info->hashDiff; + $file->xmlError = $info->xmlError; $this->revData->addFile( $info->file , $file ); diff --git a/scripts/translation/lib/RevtagParser.php b/scripts/translation/lib/RevtagParser.php index 6ca79f1a19..ed99bc01d3 100644 --- a/scripts/translation/lib/RevtagParser.php +++ b/scripts/translation/lib/RevtagParser.php @@ -33,7 +33,22 @@ class RevtagParser static function parseDir( string $lang , RevcheckFileList $list ) { foreach( $list->iterator() as $entry ) + { $entry->revtag = RevtagParser::parseFile( $lang . '/' . $entry->file ); + + // Files are parsed here anyway, so reuse the errors already + // collected by XmlUtil, instead of loading everything again. + // + // Only .xml files are checked. Entity files are DTD fragments, + // never standalone XML, so they always fail to parse as such. + + if ( str_ends_with( $entry->file , '.xml' ) == false ) + continue; + + $error = XmlUtil::$lastErrors[0] ?? null; + if ( $error != null ) + $entry->xmlError = trim( $error->message ) . " [{$error->line},{$error->column}]"; + } } public static function parseFile( string $filename ): RevtagInfo|null diff --git a/scripts/translation/lib/XmlErrorFilter.php b/scripts/translation/lib/XmlErrorFilter.php new file mode 100644 index 0000000000..a34955a0b6 --- /dev/null +++ b/scripts/translation/lib/XmlErrorFilter.php @@ -0,0 +1,116 @@ + | + * +----------------------------------------------------------------------+ + * | Description: Tell apart real XML errors from undefined entities. | + * +----------------------------------------------------------------------+ + */ + +// No require of all.php here. This file is also used by scripts/broken.php, +// that is otherwise standalone, and does not need the revcheck library. + +class XmlErrorFilter +{ + private static bool $ready = false; + private static string $prefix = ""; + private static string $suffix = ""; + private static string $extra = ""; + + /** + * Manual files are parsed one by one, without any DTD, so every entity + * reference is reported as an error by libxml. Instead of hardcoding + * message texts, that change between libxml versions, provoke the two + * known messages and learn them at runtime. + * + * - "Entity '?' not defined" (entity inside elements) + * - "Extra content at the end of the document" (entity outside elements) + */ + private static function setup() : void + { + if ( XmlErrorFilter::$ready ) + return; + + $was = libxml_use_internal_errors( true ); + + $doc = new DOMDocument(); + $doc->recover = true; + $doc->resolveExternals = false; + $doc->substituteEntities = false; + + // Setup runs lazily, on first use, so the error buffer is cleared + // around the probes below: the probe errors must not leak into the + // next document parsed, and any pending error must not be read as + // a probe result. Callers always own a copy of their own errors + // before reaching this class, so nothing of value is dropped here. + + libxml_clear_errors(); + + $doc->loadXML( "&ZZZ;" ); + $message = trim( libxml_get_errors()[0]->message ); + $message = str_replace( "ZZZ" , "\f" , $message ); + [ XmlErrorFilter::$prefix , XmlErrorFilter::$suffix ] = explode( "\f" , $message ); + libxml_clear_errors(); + + $doc->loadXML( "&ZZZ;" ); + XmlErrorFilter::$extra = trim( libxml_get_errors()[0]->message ); + libxml_clear_errors(); + + libxml_use_internal_errors( $was ); + + XmlErrorFilter::$ready = true; + } + + /** An entity reference that no DTD was around to resolve. Expected, not an error. */ + public static function isUndefinedEntity( string $message ) : bool + { + XmlErrorFilter::setup(); + $message = trim( $message ); + return str_starts_with( $message , XmlErrorFilter::$prefix ) && + str_ends_with( $message , XmlErrorFilter::$suffix ); + } + + /** Usually an entity reference outside of any enclosing tag. Reported, with a hint. */ + public static function isExtraContent( string $message ) : bool + { + XmlErrorFilter::setup(); + return trim( $message ) == XmlErrorFilter::$extra; + } + + /** + * Drop undefined entity messages, keep everything else. + * + * @param LibXMLError[] $errors + * @return LibXMLError[] + */ + public static function filter( array $errors ) : array + { + if ( count( $errors ) == 0 ) // by far the most common case + return []; + + XmlErrorFilter::setup(); + + $prefix = XmlErrorFilter::$prefix; + $suffix = XmlErrorFilter::$suffix; + + $ret = []; + foreach( $errors as $error ) + { + $message = trim( $error->message ); + if ( str_starts_with( $message , $prefix ) && str_ends_with( $message , $suffix ) ) + continue; + $ret[] = $error; + } + return $ret; + } +} diff --git a/scripts/translation/lib/XmlUtil.php b/scripts/translation/lib/XmlUtil.php index 4173ec2d08..5eab7e26b0 100644 --- a/scripts/translation/lib/XmlUtil.php +++ b/scripts/translation/lib/XmlUtil.php @@ -21,6 +21,9 @@ class XmlUtil { + /** Real errors of the last loadText(), undefined entities already filtered out. */ + public static array $lastErrors = []; + public static function extractEntities( $filename ) { $was = libxml_use_internal_errors( true ); @@ -74,7 +77,13 @@ public static function loadText( $contents ):DOMDocument $doc->resolveExternals = false; $doc->substituteEntities = false; - $doc->loadXML( $contents ); + // An empty file is a ValueError, not a parse error, and would abort + // the whole run. Feed a blank instead, so libxml reports its own + // "Document is empty" and the file is listed as broken, like any other. + + $doc->loadXML( $contents == "" ? " " : $contents ); + + XmlUtil::$lastErrors = XmlErrorFilter::filter( libxml_get_errors() ); libxml_clear_errors(); libxml_use_internal_errors( $was ); diff --git a/scripts/translation/lib/all.php b/scripts/translation/lib/all.php index 843c046bbf..3a27605a27 100644 --- a/scripts/translation/lib/all.php +++ b/scripts/translation/lib/all.php @@ -35,4 +35,5 @@ require_once __DIR__ . '/RevcheckIgnore.php'; require_once __DIR__ . '/RevcheckRun.php'; require_once __DIR__ . '/RevtagParser.php'; +require_once __DIR__ . '/XmlErrorFilter.php'; require_once __DIR__ . '/XmlUtil.php';