Add a way to configure feedback level.

docs/ioi-task-format.typ

@@ -58,6 +58,8 @@ The following are the most commonly set keys in this file:
   tasks communicate via FIFOs (by default they communicate via standard I/O,
   which is `std_io`).
   See #ref(<communication>) for further information on communication tasks.
+- `feedback_level`: set this to `oi_restricted` to instruct CMS to use IOI
+  rules for showing feedback for this task. Defaults to `full`.
 // TODO: change this to default to FIFOs with a stub and stdio otherwise?
 // TODO: IIRC stdio communication is broken, maybe fix it.
 // TODO: what about TwoStep / OutputOnly / BatchAndOutput?
@@ -214,6 +216,8 @@ Subtasks support all the keys of a group, plus:
 - `score`: the number of points for this subtask (required).
 - `validator`: overrides the default validator for this subtask.
 - `validator_args`: overrides the default validator arguments for this subtask.
+- `always_show_testcases`: whether CMS should always show feedback for all
+  testcases for this subtask, even if the subtask is not fully solved.
 
 === Testcase Definitions <testcase-definitions>
 

src/tools/export_solution_checks.rs

@@ -57,8 +57,8 @@ pub fn main_export_solution_checks(opt: ExportSolutionChecksOpt) -> Result<(), E
 
     let subtasks = task
         .subtasks
-        .iter()
-        .filter_map(|(_, info)| info.name.clone().map(|name| (name, info.id)))
+        .values()
+        .filter_map(|info| info.name.clone().map(|name| (name, info.id)))
         .collect::<HashMap<_, _>>();
 
     let checks = solutions

task-maker-cache/src/entry.rs

@@ -81,7 +81,7 @@ impl CacheEntry {
         result: Vec<ExecutionResult>,
     ) -> CacheEntry {
         let mut items = Vec::new();
-        for (exec, res) in group.executions.iter().zip(result.into_iter()) {
+        for (exec, res) in group.executions.iter().zip(result) {
             items.push(CacheEntryItem::from_execution(exec, file_keys, res));
         }
         CacheEntry {
@@ -163,10 +163,8 @@ impl CacheEntry {
         macro_rules! check_limit {
             ($left:expr, $right:expr, $extra_time:expr) => {
                 match ($left, $right) {
-                    (Some(left), Some(right)) => {
-                        if left + $extra_time > right {
-                            return false;
-                        }
+                    (Some(left), Some(right)) if left + $extra_time > right => {
+                        return false;
                     }
                     (None, Some(_)) => return false,
                     _ => {}

task-maker-cache/src/lib.rs

@@ -177,25 +177,24 @@ impl Cache {
                 None => {
                     // TODO: remove the entry because it's not valid anymore
                 }
-                Some(outputs) => {
-                    if entry.is_compatible(group) {
-                        let mut results = Vec::new();
-                        for (exec, item) in group.executions.iter().zip(entry.items.iter()) {
-                            results.push(ExecutionResult {
-                                status: exec.status(&item.result.status, &item.result.resources),
-                                was_killed: item.result.was_killed,
-                                was_cached: true,
-                                resources: item.result.resources.clone(),
-                                stdout: item.result.stdout.clone(),
-                                stderr: item.result.stderr.clone(),
-                            });
-                        }
-                        return CacheResult::Hit {
-                            result: results,
-                            outputs,
-                        };
+                Some(outputs) if entry.is_compatible(group) => {
+                    let mut results = Vec::new();
+                    for (exec, item) in group.executions.iter().zip(entry.items.iter()) {
+                        results.push(ExecutionResult {
+                            status: exec.status(&item.result.status, &item.result.resources),
+                            was_killed: item.result.was_killed,
+                            was_cached: true,
+                            resources: item.result.resources.clone(),
+                            stdout: item.result.stdout.clone(),
+                            stderr: item.result.stderr.clone(),
+                        });
                     }
+                    return CacheResult::Hit {
+                        result: results,
+                        outputs,
+                    };
                 }
+                _ => {}
             }
         }
         CacheResult::Miss

task-maker-format/src/ioi/format/italian_toml/gen_toml.rs

@@ -172,6 +172,9 @@ impl GroupConfig {
 struct SubtaskConfig {
     score: f64,
 
+    #[serde(default)]
+    always_show_testcases: bool,
+
     #[serde(flatten)]
     group: GroupConfig,
 
@@ -473,6 +476,7 @@ pub(super) fn parse(
                 testcases,
                 testcases_owned,
                 input_validator,
+                always_show_testcases: subtask.always_show_testcases,
                 ..Default::default()
             },
         );

task-maker-format/src/ioi/format/italian_toml/mod.rs

@@ -8,7 +8,7 @@ use itertools::Itertools;
 use task_maker_lang::GraderMap;
 
 use super::italian_yaml::TaskYAML;
-use crate::ioi::italian_yaml::{TaskYAMLOrig, TM_ALLOW_DELETE_COOKIE};
+use crate::ioi::italian_yaml::{ScoreTypeGroupParameters, TaskYAMLOrig, TM_ALLOW_DELETE_COOKIE};
 use crate::ioi::sanity_checks::get_sanity_checks;
 use crate::ioi::{
     make_task_booklets, BatchTypeData, Checker, CommunicationTypeData, IOITask,
@@ -85,15 +85,19 @@ pub fn parse_task<P: AsRef<Path>>(
                         .testcases
                         .iter()
                         .map(|tc_num| format!("{tc_num:03}"))
-                        .join("|");
-                    (st.max_score, testcases)
+                        .collect();
+                    ScoreTypeGroupParameters::Dict {
+                        max_score: st.max_score,
+                        testcases,
+                        always_show_testcases: st.always_show_testcases,
+                    }
                 })
                 .collect(),
         );
 
         let n_input = subtasks
-            .iter()
-            .flat_map(|(_, st)| st.testcases.clone())
+            .values()
+            .flat_map(|st| st.testcases.clone())
             .unique()
             .count();
         config.n_input = Some(n_input);

task-maker-format/src/ioi/format/italian_yaml/mod.rs

@@ -330,6 +330,17 @@ where
     }
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(untagged)]
+pub(super) enum ScoreTypeGroupParameters {
+    List((f64, String)),
+    Dict {
+        max_score: f64,
+        testcases: Vec<String>,
+        always_show_testcases: bool,
+    },
+}
+
 /// Deserialized data from the task.yaml of a IOI format task.
 #[derive(Debug, Serialize, Deserialize)]
 pub(super) struct TaskYAML {
@@ -344,7 +355,7 @@ pub(super) struct TaskYAML {
     pub score_type: Option<TestcaseScoreAggregator>,
     /// The parameters of the score type.
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub score_type_parameters: Option<Vec<(f64, String)>>,
+    pub score_type_parameters: Option<Vec<ScoreTypeGroupParameters>>,
     /// The number of inputs of the task
     #[serde(skip_serializing_if = "Option::is_none")]
     pub n_input: Option<usize>,
@@ -404,7 +415,7 @@ pub(super) struct TaskYAML {
     pub token_mode: Option<String>,
     /// Compatibility with cms, unused.
     pub public_testcases: Option<String>,
-    /// Compatibility with cms, unused.
+    /// Compatibility with cms, not directly used.
     pub feedback_level: Option<String>,
 }
 
@@ -465,6 +476,9 @@ pub(super) struct TaskYAMLOrig {
     /// Whether the solution processes are assumed to be concurrent (wall time max-ed, memory summed)
     /// or sequential (wall time sum-ed, memory max-ed).
     pub interactive_concurrent: Option<bool>,
+
+    /// Compatibility with cms, not directly used.
+    pub feedback_level: Option<String>,
 }
 
 impl TaskYAMLOrig {
@@ -501,7 +515,7 @@ impl TaskYAMLOrig {
             score_mode: Some("max_subtask".into()),
             token_mode: Some("disabled".into()),
             public_testcases: Some("all".into()),
-            feedback_level: Some("full".into()),
+            feedback_level: Some(self.feedback_level.clone().unwrap_or("full".to_string())),
         }
     }
 }
@@ -697,14 +711,14 @@ pub fn parse_task<P: AsRef<Path>>(
                                 .iter()
                                 .map(|tc_num| format!("{tc_num:03}"))
                                 .join("|");
-                            (st.max_score, testcases)
+                            ScoreTypeGroupParameters::List((st.max_score, testcases))
                         })
                         .collect(),
                 );
 
                 let n_input = subtasks
-                    .iter()
-                    .flat_map(|(_, st)| st.testcases.clone())
+                    .values()
+                    .flat_map(|st| st.testcases.clone())
                     .unique()
                     .count();
                 yaml.n_input = Some(n_input);

task-maker-format/src/ioi/mod.rs

@@ -178,6 +178,8 @@ pub struct SubtaskInfo {
     pub is_default: bool,
     /// The list of the dependencies of this subtask.
     pub dependencies: Vec<SubtaskId>,
+    /// Whether we should ask CMS to always show testcase information for this subtask.
+    pub always_show_testcases: bool,
 }
 
 /// A testcase of a IOI task.