diff --git a/.vscode/dats.code-workspace b/.vscode/dats.code-workspace index a0e6b5b78..c2d4b7f77 100644 --- a/.vscode/dats.code-workspace +++ b/.vscode/dats.code-workspace @@ -7,6 +7,10 @@ { "name": "backend", "path": "../backend", + }, + { + "name": "benchmarks", + "path": "../benchmarks", }, { "name": "ray", diff --git a/.vscode/settings.json b/.vscode/settings.json index ca5252035..4e87e5a64 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,6 +3,7 @@ "files.exclude": { "airflow": true, "backend": true, + "benchmarks": true, "ray": true, "frontend": true }, diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore new file mode 100644 index 000000000..703b548e7 --- /dev/null +++ b/benchmarks/.gitignore @@ -0,0 +1,14 @@ +.venv/ +__pycache__/ +*.pyc + +# Local env overrides +docker/.env +src/.env + +# Generated benchmark artifacts +outputs/*.csv +outputs/*.json +outputs/*.png +data/**/*.csv +run_experiment.log diff --git a/benchmarks/.vscode/launch.json b/benchmarks/.vscode/launch.json new file mode 100644 index 000000000..0a54b0c3b --- /dev/null +++ b/benchmarks/.vscode/launch.json @@ -0,0 +1,95 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "fastapi", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/src/main.py", + "console": "integratedTerminal", + "justMyCode": true, + "cwd": "${workspaceFolder}", + "envFile": "${workspaceFolder}/.env", + "env": { + "PYTHONPATH": "${workspaceFolder}/src" + } + }, + { + "name": "rq", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/src/worker.py", + "args": ["work", "dev"], + "console": "integratedTerminal", + "justMyCode": true, + "cwd": "${workspaceFolder}", + "envFile": "${workspaceFolder}/.env", + "env": { + "PYTHONPATH": "${workspaceFolder}/src", + "RQ_WORKERS_CPU": "1", + "RQ_WORKERS_API": "1", + "RQ_WORKERS_GPU": "1" + } + }, + { + "name": "pytest", + "type": "debugpy", + "request": "launch", + "module": "pytest", + "console": "integratedTerminal", + "justMyCode": true, + "cwd": "${workspaceFolder}", + "envFile": "${workspaceFolder}/.env", + "env": { + "PYTHONPATH": "${workspaceFolder}/src", + "RESET_DATABASE_FOR_TESTING": "1" + } + }, + { + "name": "pyright", + "type": "node-terminal", + "request": "launch", + "command": "uv run pyright", + "cwd": "${workspaceFolder}", + "envFile": "${workspaceFolder}/.env", + "env": { + "PYTHONPATH": "${workspaceFolder}/src" + } + }, + { + "name": "Alembic: migrate", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/src/migrations/run_migrations.py", + "console": "integratedTerminal", + "justMyCode": true, + "cwd": "${workspaceFolder}", + "envFile": "${workspaceFolder}/.env", + "env": { + "PYTHONPATH": "${workspaceFolder}/src" + } + }, + { + "name": "Alembic: check", + "type": "node-terminal", + "request": "launch", + "command": "uv run alembic -c src/migrations/alembic.ini check", + "cwd": "${workspaceFolder}", + "envFile": "${workspaceFolder}/.env", + "env": { + "PYTHONPATH": "${workspaceFolder}/src" + } + }, + { + "name": "Alembic: revision", + "type": "node-terminal", + "request": "launch", + "command": "uv run alembic -c src/migrations/alembic.ini revision --autogenerate -m \"vscode launcher\"", + "cwd": "${workspaceFolder}", + "envFile": "${workspaceFolder}/.env", + "env": { + "PYTHONPATH": "${workspaceFolder}/src" + } + } + ] +} diff --git a/benchmarks/.vscode/settings.json b/benchmarks/.vscode/settings.json new file mode 100644 index 000000000..2b76b7a7d --- /dev/null +++ b/benchmarks/.vscode/settings.json @@ -0,0 +1,16 @@ +{ + // python + "python.defaultInterpreterPath": "${workspaceFolder:benchmarks}/.venv/bin/python", + "python.envFile": "${workspaceFolder:benchmarks}/.env", + "python.autoComplete.extraPaths": ["${workspaceFolder:benchmarks}/src"], + "python.analysis.extraPaths": ["${workspaceFolder:benchmarks}/src"], + "python.analysis.pyrightVersion": "1.1.385", // this has to match pyproject.toml + "python.analysis.exclude": ["**/__pycache__", "**/.venv"], + + // prettier + "prettier.prettierPath": "../frontend/node_modules/prettier", + "prettier.configPath": "../.prettierrc.yaml", + + // ruff + "ruff.interpreter": ["${workspaceFolder:benchmarks}/.venv/bin/python"] +} diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..4f7e4cb6f --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,62 @@ +# LLM Benchmarking Framework + +This folder contains a modular and reproducible framework for benchmarking LLMs across NLP tasks. + +## First Working Experiment + +The first implemented end-to-end experiment is document classification on a sampled 20 Newsgroups split. + +Configuration rules: + +- Experiment and backend configs are composed via Hydra groups into typed `RunConfig` as `experiment` and `backend`. +- Model config is nested in each experiment via `defaults` (`/model: ...`) as `experiment.model`. +- Dataset config is nested in each experiment via `defaults` (`/dataset: ...`) as `experiment.dataset`. +- Dataset configs define `name`, `path`, `text_column`, and `label_column`. +- `run_name` in experiment configs is optional. If omitted, MLflow auto-generates it. +- Prompt templates are always loaded from `src/prompts/templates` (not configurable). +- Schema is configured as a single dotted path (for example `newsgroups20_schema.NewsgroupClassificationSchemaV1`). + +### 1. Install dependencies + +```bash +cd benchmarks +uv sync +``` + +### 2. Prepare data + +```bash +uv run python data/20newsgroups/preprocess.py +``` + +### 3. Start MLflow service + +```bash +cd docker +cp .env.example .env +docker compose up -d +cd .. +``` + +### 4. Run the 20 Newsgroups experiment + +```bash +uv run python src/run_experiment.py +``` + +### 5. Override config groups (example) + +```bash +uv run python src/run_experiment.py \ + experiment=20newsgroups_v1_zeroshot \ + backend=vllm \ + backend.gpu_id=1 +``` + +## Layout + +- `configs/`: Runtime config and Hydra groups (`experiment/`, `model/`, `dataset/`, `backend/`) +- `data/`: Datasets and preprocessing scripts +- `docker/`: MLflow compose files and environment templates +- `outputs/`: Local output artifacts (CSV/JSON) +- `src/`: Core runner, LLM clients, schemas, prompts, evaluation, tracking diff --git a/benchmarks/configs/backend/vllm.yaml b/benchmarks/configs/backend/vllm.yaml new file mode 100644 index 000000000..a243a1611 --- /dev/null +++ b/benchmarks/configs/backend/vllm.yaml @@ -0,0 +1,8 @@ +image: vllm/vllm-openai:latest +host_port: 19275 +startup_timeout_seconds: 600 +gpu_id: 1 +hf_token_env_var: HF_TOKEN +hf_cache_dir: ~/.cache/huggingface +concurrency: 24 +api_key: EMPTY diff --git a/benchmarks/configs/config.yaml b/benchmarks/configs/config.yaml new file mode 100644 index 000000000..e4f0179cb --- /dev/null +++ b/benchmarks/configs/config.yaml @@ -0,0 +1,15 @@ +defaults: + - experiment: 20newsgroups_v1_zeroshot + - backend: vllm + - _self_ + +output_dir: outputs +mlflow_uri: http://localhost:19274 +fail_on_parse_error: false + +hydra: + job: + chdir: false + run: + dir: . + output_subdir: null diff --git a/benchmarks/configs/dataset/20newsgroups.yaml b/benchmarks/configs/dataset/20newsgroups.yaml new file mode 100644 index 000000000..96a1b0111 --- /dev/null +++ b/benchmarks/configs/dataset/20newsgroups.yaml @@ -0,0 +1,5 @@ +name: 20newsgroups +dataset_type: document_classification_single_label +path: 20newsgroups/test_full_raw.parquet +text_column: document_text +label_column: label diff --git a/benchmarks/configs/dataset/bbc-coarse.yaml b/benchmarks/configs/dataset/bbc-coarse.yaml new file mode 100644 index 000000000..a30bad3e8 --- /dev/null +++ b/benchmarks/configs/dataset/bbc-coarse.yaml @@ -0,0 +1,5 @@ +name: bbc-coarse +dataset_type: document_classification_single_label +path: bbc/bbc_cleaned.parquet +text_column: content +label_column: main_tag diff --git a/benchmarks/configs/dataset/bbc-fine.yaml b/benchmarks/configs/dataset/bbc-fine.yaml new file mode 100644 index 000000000..a549afdaf --- /dev/null +++ b/benchmarks/configs/dataset/bbc-fine.yaml @@ -0,0 +1,5 @@ +name: bbc-fine +dataset_type: document_classification_single_label +path: bbc/bbc_cleaned.parquet +text_column: content +label_column: tag diff --git a/benchmarks/configs/dataset/coarsediscourse.yaml b/benchmarks/configs/dataset/coarsediscourse.yaml new file mode 100644 index 000000000..eb0273056 --- /dev/null +++ b/benchmarks/configs/dataset/coarsediscourse.yaml @@ -0,0 +1,5 @@ +name: coarsediscourse +dataset_type: sequential_sentence_classification +path: coarsediscourse/coursediscourse_test.parquet +sentences_column: sentences +labels_column: labels diff --git a/benchmarks/configs/dataset/csabstruct.yaml b/benchmarks/configs/dataset/csabstruct.yaml new file mode 100644 index 000000000..0784c642c --- /dev/null +++ b/benchmarks/configs/dataset/csabstruct.yaml @@ -0,0 +1,5 @@ +name: csabstruct +dataset_type: sequential_sentence_classification +path: csabstruct/test.parquet +sentences_column: sentences +labels_column: labels diff --git a/benchmarks/configs/dataset/daily-dialog.yaml b/benchmarks/configs/dataset/daily-dialog.yaml new file mode 100644 index 000000000..b72f25c8a --- /dev/null +++ b/benchmarks/configs/dataset/daily-dialog.yaml @@ -0,0 +1,5 @@ +name: daily-dialog +dataset_type: sequential_sentence_classification +path: daily_dialog/dailydialog_test.parquet +sentences_column: sentences +labels_column: labels diff --git a/benchmarks/configs/dataset/emotion-lines.yaml b/benchmarks/configs/dataset/emotion-lines.yaml new file mode 100644 index 000000000..2ce68c147 --- /dev/null +++ b/benchmarks/configs/dataset/emotion-lines.yaml @@ -0,0 +1,7 @@ +name: emotion-lines +dataset_type: sequential_sentence_classification +path: emotion_lines/friends_test.parquet +sentences_column: sentences +labels_column: labels +unwanted_labels: + - non-neutral diff --git a/benchmarks/configs/dataset/fewnerd-coarse.yaml b/benchmarks/configs/dataset/fewnerd-coarse.yaml new file mode 100644 index 000000000..fc1b1eae8 --- /dev/null +++ b/benchmarks/configs/dataset/fewnerd-coarse.yaml @@ -0,0 +1,15 @@ +name: fewnerd-coarse +dataset_type: span_classification +path: fewnerd/fewnerd_test.parquet +tokens_column: tokens +tags_column: ner_tags +id2label: + 0: O + 1: art + 2: building + 3: event + 4: location + 5: organization + 6: other + 7: person + 8: product diff --git a/benchmarks/configs/dataset/fewnerd-fine.yaml b/benchmarks/configs/dataset/fewnerd-fine.yaml new file mode 100644 index 000000000..02ca6947a --- /dev/null +++ b/benchmarks/configs/dataset/fewnerd-fine.yaml @@ -0,0 +1,73 @@ +name: fewnerd-fine +dataset_type: span_classification +path: fewnerd/fewnerd_test.parquet +tokens_column: tokens +tags_column: fine_ner_tags +id2label: + 0: O + 1: art - broadcastprogram + 2: art - film + 3: art - music + 4: art - other + 5: art - painting + 6: art - writtenart + 7: building - airport + 8: building - hospital + 9: building - hotel + 10: building - library + 11: building - other + 12: building - restaurant + 13: building - sportsfacility + 14: building - theater + 15: event - attack/battle/war/militaryconflict + 16: event - disaster + 17: event - election + 18: event - other + 19: event - protest + 20: event - sportsevent + 21: location - GPE + 22: location - bodiesofwater + 23: location - island + 24: location - mountain + 25: location - other + 26: location - park + 27: location - road/railway/highway/transit + 28: organization - company + 29: organization - education + 30: organization - government/governmentagency + 31: organization - media/newspaper + 32: organization - other + 33: organization - politicalparty + 34: organization - religion + 35: organization - showorganization + 36: organization - sportsleague + 37: organization - sportsteam + 38: other - astronomything + 39: other - award + 40: other - biologything + 41: other - chemicalthing + 42: other - currency + 43: other - disease + 44: other - educationaldegree + 45: other - god + 46: other - language + 47: other - law + 48: other - livingthing + 49: other - medical + 50: person - actor + 51: person - artist/author + 52: person - athlete + 53: person - director + 54: person - other + 55: person - politician + 56: person - scholar + 57: person - soldier + 58: product - airplane + 59: product - car + 60: product - food + 61: product - game + 62: product - other + 63: product - ship + 64: product - software + 65: product - train + 66: product - weapon diff --git a/benchmarks/configs/dataset/german-ler-coarse.yaml b/benchmarks/configs/dataset/german-ler-coarse.yaml new file mode 100644 index 000000000..44cf7edbe --- /dev/null +++ b/benchmarks/configs/dataset/german-ler-coarse.yaml @@ -0,0 +1,14 @@ +name: german-ler-coarse +dataset_type: span_classification +path: german-ler/german_ler_test.parquet +tokens_column: tokens +tags_column: ner_tags +id2label: + 0: O + 1: person + 2: ort + 3: organisation + 4: norm + 5: gesetz + 6: rechtsprechung + 7: literatur diff --git a/benchmarks/configs/dataset/german-ler-fine.yaml b/benchmarks/configs/dataset/german-ler-fine.yaml new file mode 100644 index 000000000..ba18b5d86 --- /dev/null +++ b/benchmarks/configs/dataset/german-ler-fine.yaml @@ -0,0 +1,26 @@ +name: german-ler-fine +dataset_type: span_classification +path: german-ler/german_ler_test.parquet +tokens_column: tokens +tags_column: fine_ner_tags +id2label: + 0: O + 1: Person + 2: Anwalt + 3: Richter + 4: Land + 5: Stadt + 6: Straße + 7: Landschaft + 8: Organisation + 9: Unternehmen + 10: Institution + 11: Gericht + 12: Marke + 13: Gesetz + 14: Verordnung + 15: EU Norm + 16: Vorschrift + 17: Vertrag + 18: Gerichtsentscheidung + 19: Literatur diff --git a/benchmarks/configs/dataset/german-quotations-direct.yaml b/benchmarks/configs/dataset/german-quotations-direct.yaml new file mode 100644 index 000000000..a138a7c6d --- /dev/null +++ b/benchmarks/configs/dataset/german-quotations-direct.yaml @@ -0,0 +1,9 @@ +name: german-quotations-direct +dataset_type: span_classification +path: german-quotations/german_direct_quotations.parquet +tokens_column: tokens +tags_column: tags +id2label: + 0: O + 1: Sprecher + 2: Direkte Rede diff --git a/benchmarks/configs/dataset/germanquad.yaml b/benchmarks/configs/dataset/germanquad.yaml new file mode 100644 index 000000000..86fcff769 --- /dev/null +++ b/benchmarks/configs/dataset/germanquad.yaml @@ -0,0 +1,6 @@ +name: germanquad +dataset_type: extractive_qa +path: germanquad/test.parquet +context_column: context +question_column: question +references_column: reference diff --git a/benchmarks/configs/dataset/imdb-coarse.yaml b/benchmarks/configs/dataset/imdb-coarse.yaml new file mode 100644 index 000000000..030be214d --- /dev/null +++ b/benchmarks/configs/dataset/imdb-coarse.yaml @@ -0,0 +1,5 @@ +name: imdb-coarse +dataset_type: document_classification_single_label +path: imdb/imdb_cleaned.parquet +text_column: description +label_column: genre diff --git a/benchmarks/configs/dataset/imdb-multi-label.yaml b/benchmarks/configs/dataset/imdb-multi-label.yaml new file mode 100644 index 000000000..38f93d871 --- /dev/null +++ b/benchmarks/configs/dataset/imdb-multi-label.yaml @@ -0,0 +1,5 @@ +name: imdb-multi-label +dataset_type: document_classification_multi_label +path: imdb/imdb_cleaned.parquet +text_column: description +label_column: expanded-genres diff --git a/benchmarks/configs/dataset/muc4.yaml b/benchmarks/configs/dataset/muc4.yaml new file mode 100644 index 000000000..2c2adf12f --- /dev/null +++ b/benchmarks/configs/dataset/muc4.yaml @@ -0,0 +1,11 @@ +name: muc4 +dataset_type: template_filling +path: muc/muc.parquet +context_column: doctext +slot_columns: + incident: incident + perpetrator: perpetrator + group_perpetrator: group perpetrator + victim: victim + target: target + weapon: weapon diff --git a/benchmarks/configs/dataset/pubmed200k.yaml b/benchmarks/configs/dataset/pubmed200k.yaml new file mode 100644 index 000000000..0b3f67611 --- /dev/null +++ b/benchmarks/configs/dataset/pubmed200k.yaml @@ -0,0 +1,5 @@ +name: pubmed200k +dataset_type: sequential_sentence_classification +path: pubmed200k/test.parquet +sentences_column: sentences +labels_column: labels diff --git a/benchmarks/configs/dataset/squad.yaml b/benchmarks/configs/dataset/squad.yaml new file mode 100644 index 000000000..ddc92263b --- /dev/null +++ b/benchmarks/configs/dataset/squad.yaml @@ -0,0 +1,6 @@ +name: squad +dataset_type: extractive_qa +path: squad/validation.parquet +context_column: context +question_column: question +references_column: reference diff --git a/benchmarks/configs/dataset/squad2.yaml b/benchmarks/configs/dataset/squad2.yaml new file mode 100644 index 000000000..f2722a375 --- /dev/null +++ b/benchmarks/configs/dataset/squad2.yaml @@ -0,0 +1,6 @@ +name: squad2 +dataset_type: extractive_qa +path: squad2/validation.parquet +context_column: context +question_column: question +references_column: reference diff --git a/benchmarks/configs/dataset/tagesschau-coarse.yaml b/benchmarks/configs/dataset/tagesschau-coarse.yaml new file mode 100644 index 000000000..17286e74c --- /dev/null +++ b/benchmarks/configs/dataset/tagesschau-coarse.yaml @@ -0,0 +1,5 @@ +name: tagesschau-coarse +dataset_type: document_classification_single_label +path: tagesschau/tagesschau_cleaned.parquet +text_column: article +label_column: main_tag diff --git a/benchmarks/configs/dataset/tagesschau-fine.yaml b/benchmarks/configs/dataset/tagesschau-fine.yaml new file mode 100644 index 000000000..2391a5cbf --- /dev/null +++ b/benchmarks/configs/dataset/tagesschau-fine.yaml @@ -0,0 +1,5 @@ +name: tagesschau-fine +dataset_type: document_classification_single_label +path: tagesschau/tagesschau_cleaned.parquet +text_column: article +label_column: tag diff --git a/benchmarks/configs/experiment/document_classification_20newsgroups_coarse_v1.yaml b/benchmarks/configs/experiment/document_classification_20newsgroups_coarse_v1.yaml new file mode 100644 index 000000000..8666d723b --- /dev/null +++ b/benchmarks/configs/experiment/document_classification_20newsgroups_coarse_v1.yaml @@ -0,0 +1,23 @@ +defaults: + - /model: llama32_3b + - /dataset: 20newsgroups + +experiment_name: document_classification_coarse +# run_name: llama32_3b_20newsgroups_zeroshot_v1 + +max_examples: 120 + +prompt_template: document_classification_single_label_v1_en.j2 +prompt_variables: + labels: + - alt.atheism + - comp.graphics + - sci.space + - talk.politics.mideast + +answer_schema: newsgroups20_schema.NewsgroupClassificationSchemaV1 + +metrics: + - classification_macro_metrics + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/document_classification_bbc_coarse_v1.yaml b/benchmarks/configs/experiment/document_classification_bbc_coarse_v1.yaml new file mode 100644 index 000000000..8d59a0203 --- /dev/null +++ b/benchmarks/configs/experiment/document_classification_bbc_coarse_v1.yaml @@ -0,0 +1,35 @@ +defaults: + - /model: llama32_3b + - /dataset: bbc-coarse + +experiment_name: document_classification_coarse +# run_name: llama32_3b_bbc_coarse_v1 + +max_examples: 10000 +sample_randomly: true +sample_random_state: 42 + +prompt_template: document_classification_single_label_v1_en.j2 +prompt_variables: + labels: + - uk - News about the United Kingdom, including politics and domestic topics. + - world - News about regions outside of the UK. + - sport - News about all kinds of sports. + - misc - Other news such as business, education, entertainment, health, science, and technology. + +system_prompt_template: document_classification_system_en.j2 +system_prompt_variables: + project_name: BBC + project_description: An analysis of topics discussed in the United Kingdom based on BBC news and articles. + +answer_schema: document_classification_schema.BBCCoarseClassificationSchemaV1 + +metrics: + - classification_macro_metrics + - classification_weighted_metrics + +artifacts: + - classification_confusion_matrix + - classification_report + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/document_classification_bbc_fine_v1.yaml b/benchmarks/configs/experiment/document_classification_bbc_fine_v1.yaml new file mode 100644 index 000000000..3fbab4643 --- /dev/null +++ b/benchmarks/configs/experiment/document_classification_bbc_fine_v1.yaml @@ -0,0 +1,57 @@ +defaults: + - /model: llama32_3b + - /dataset: bbc-fine + +experiment_name: document_classification_fine +# run_name: llama32_3b_bbc_fine_v1 + +max_examples: 10000 +sample_randomly: true +sample_random_state: 42 + +prompt_template: document_classification_single_label_v1_en.j2 +prompt_variables: + labels: + - uk/england + - uk/scotland + - uk/wales + - uk/northern-ireland + - uk/politics + - world/africa + - world/asia + - world/australia + - world/europe + - world/latin-america + - world/middle-east + - world/us + - sport/athletics + - sport/boxing + - sport/cricket + - sport/football + - sport/formula1 + - sport/rugby + - sport/tennis + - misc/business + - misc/education + - misc/election + - misc/entertainment + - misc/health + - misc/science + - misc/technology + +system_prompt_template: document_classification_system_en.j2 +system_prompt_variables: + project_name: BBC + project_description: An analysis of topics discussed in the United Kingdom based on BBC news and articles. + +answer_schema: document_classification_schema.BBCFineClassificationSchemaV1 + +metrics: + - classification_macro_metrics + - classification_weighted_metrics + +artifacts: + - classification_confusion_matrix + - classification_report + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/document_classification_imdb_coarse_v1.yaml b/benchmarks/configs/experiment/document_classification_imdb_coarse_v1.yaml new file mode 100644 index 000000000..d6723d7bc --- /dev/null +++ b/benchmarks/configs/experiment/document_classification_imdb_coarse_v1.yaml @@ -0,0 +1,47 @@ +defaults: + - /model: llama32_3b + - /dataset: imdb-coarse + +experiment_name: document_classification_coarse +# run_name: llama32_3b_imdb_coarse_v1 + +max_examples: 10000 +sample_randomly: true +sample_random_state: 42 + +prompt_template: document_classification_single_label_v1_en.j2 +prompt_variables: + labels: + - action + - adventure + - animation + - biography + - crime + - family + - fantasy + - film-noir + - history + - horror + - mystery + - romance + - scifi + - sports + - thriller + - war + +system_prompt_template: document_classification_system_en.j2 +system_prompt_variables: + project_name: Movie Genres + project_description: An analysis of movie genres based on IMDb movie descriptions. + +answer_schema: document_classification_schema.IMDBCoarseClassificationSchemaV1 + +metrics: + - classification_macro_metrics + - classification_weighted_metrics + +artifacts: + - classification_confusion_matrix + - classification_report + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/document_classification_imdb_multi_label_v1.yaml b/benchmarks/configs/experiment/document_classification_imdb_multi_label_v1.yaml new file mode 100644 index 000000000..159cb3a4f --- /dev/null +++ b/benchmarks/configs/experiment/document_classification_imdb_multi_label_v1.yaml @@ -0,0 +1,55 @@ +defaults: + - /model: llama32_3b + - /dataset: imdb-multi-label + +experiment_name: document_classification_multi_label +# run_name: llama32_3b_imdb_multi_label_v1 + +max_examples: 10000 +sample_randomly: true +sample_random_state: 42 + +prompt_template: document_classification_multi_label_v1_en.j2 +prompt_variables: + labels: + - action + - adventure + - animation + - biography + - comedy + - crime + - drama + - family + - fantasy + - film-noir + - game-show + - history + - horror + - music + - musical + - mystery + - news + - reality-tv + - romance + - sci-fi + - sport + - talk-show + - thriller + - war + - western + +system_prompt_template: document_classification_system_en.j2 +system_prompt_variables: + project_name: Movie Genres + project_description: An analysis of movie genres based on IMDb movie descriptions. + +answer_schema: document_classification_schema.IMDBMultiLabelClassificationSchemaV1 + +metrics: + - multilabel_weighted_metrics + +artifacts: + - multilabel_confusion_matrices + - multilabel_classification_report + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/document_classification_tagesschau_coarse_v1.yaml b/benchmarks/configs/experiment/document_classification_tagesschau_coarse_v1.yaml new file mode 100644 index 000000000..2b9991bed --- /dev/null +++ b/benchmarks/configs/experiment/document_classification_tagesschau_coarse_v1.yaml @@ -0,0 +1,35 @@ +defaults: + - /model: llama32_3b + - /dataset: tagesschau-coarse + +experiment_name: document_classification_coarse +# run_name: llama32_3b_tagesschau_coarse_v1 + +max_examples: 100 +sample_randomly: true +sample_random_state: 42 + +prompt_template: document_classification_single_label_v1_de.j2 +prompt_variables: + labels: + - inland - Nachrichten über Deutschland. + - ausland - Nachrichten über das Ausland. + - wirtschaft - Nachrichten über Wirtschaft, Börse und Unternehmen. + - wissen - Nachrichten über Wissenschaft, Forschung, Gesundheit und Klima. + +system_prompt_template: document_classification_system_de.j2 +system_prompt_variables: + project_name: Tagesschau + project_description: Eine Analyse der Themen in Deutschland basierend auf den Nachrichten und Artikeln der Tagesschau. + +answer_schema: document_classification_schema.TagesschauCoarseClassificationSchemaV1 + +metrics: + - classification_macro_metrics + - classification_weighted_metrics + +artifacts: + - classification_confusion_matrix + - classification_report + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/document_classification_tagesschau_fine_v1.yaml b/benchmarks/configs/experiment/document_classification_tagesschau_fine_v1.yaml new file mode 100644 index 000000000..6b60e7e3a --- /dev/null +++ b/benchmarks/configs/experiment/document_classification_tagesschau_fine_v1.yaml @@ -0,0 +1,51 @@ +defaults: + - /model: llama32_3b + - /dataset: tagesschau-fine + +experiment_name: document_classification_fine +# run_name: llama32_3b_tagesschau_fine_v1 + +max_examples: 10000 +sample_randomly: true +sample_random_state: 42 + +prompt_template: document_classification_single_label_v1_de.j2 +prompt_variables: + labels: + - inland/deutschlandtrend + - inland/gesellschaft + - inland/innenpolitik + - inland/mittendrin + - ausland/afrika + - ausland/amerika + - ausland/asien + - ausland/europa + - ausland/ozeanien + - wirtschaft/boerse + - wirtschaft/finanzen + - wirtschaft/konjunktur + - wirtschaft/technologie + - wirtschaft/unternehmen + - wirtschaft/verbraucher + - wirtschaft/weltwirtschaft + - wissen/forschung + - wissen/gesundheit + - wissen/klima + - wissen/technologie + +system_prompt_template: document_classification_system_de.j2 +system_prompt_variables: + project_name: Tagesschau + project_description: Eine Analyse der Themen in Deutschland basierend auf den Nachrichten und Artikeln der Tagesschau. + +answer_schema: document_classification_schema.TagesschauFineClassificationSchemaV1 + +metrics: + - classification_macro_metrics + - classification_weighted_metrics + +artifacts: + - classification_confusion_matrix + - classification_report + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/extractive_qa_germanquad_v1.yaml b/benchmarks/configs/experiment/extractive_qa_germanquad_v1.yaml new file mode 100644 index 000000000..ff631ca48 --- /dev/null +++ b/benchmarks/configs/experiment/extractive_qa_germanquad_v1.yaml @@ -0,0 +1,17 @@ +defaults: + - /model: llama32_3b + - /dataset: germanquad + +experiment_name: extractive_qa +# run_name: llama32_3b_germanquad_v1 + +prompt_template: extractive_qa_v1_de.j2 + +system_prompt_template: extractive_qa_system_de.j2 + +answer_schema: extractive_qa_schema.ExtractiveQAAnswerSchemaV1 + +metrics: + - extractive_qa_squad2_metrics + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/extractive_qa_squad1_v1.yaml b/benchmarks/configs/experiment/extractive_qa_squad1_v1.yaml new file mode 100644 index 000000000..12b85754a --- /dev/null +++ b/benchmarks/configs/experiment/extractive_qa_squad1_v1.yaml @@ -0,0 +1,21 @@ +defaults: + - /model: llama32_3b + - /dataset: squad + +experiment_name: extractive_qa +# run_name: llama32_3b_squad1_v1 + +max_examples: 1000 +sample_randomly: true +sample_random_state: 42 + +prompt_template: extractive_qa_squad1_v1_en.j2 + +system_prompt_template: extractive_qa_system_en.j2 + +answer_schema: extractive_qa_schema.ExtractiveQAAnswerSchemaV1 + +metrics: + - extractive_qa_squad2_metrics + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/extractive_qa_squad2_v1.yaml b/benchmarks/configs/experiment/extractive_qa_squad2_v1.yaml new file mode 100644 index 000000000..1e69e2ea5 --- /dev/null +++ b/benchmarks/configs/experiment/extractive_qa_squad2_v1.yaml @@ -0,0 +1,19 @@ +defaults: + - /model: llama32_3b + - /dataset: squad2 + +experiment_name: extractive_qa +# run_name: llama32_3b_squad2_v1 + +prompt_template: extractive_qa_v1_en.j2 +prompt_variables: + no_answer_label: Not answerable + +system_prompt_template: extractive_qa_system_en.j2 + +answer_schema: extractive_qa_schema.ExtractiveQAAnswerSchemaV1 + +metrics: + - extractive_qa_squad2_metrics + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/sequential_sentence_classification_coarsediscourse_v1.yaml b/benchmarks/configs/experiment/sequential_sentence_classification_coarsediscourse_v1.yaml new file mode 100644 index 000000000..8c4e73bc4 --- /dev/null +++ b/benchmarks/configs/experiment/sequential_sentence_classification_coarsediscourse_v1.yaml @@ -0,0 +1,50 @@ +defaults: + - /model: llama32_3b + - /dataset: coarsediscourse + +experiment_name: sequential_sentence_classification +# run_name: llama32_3b_coarsediscourse_zeroshot_v1 + +max_examples: 500 + +prompt_template: sequential_sentence_classification_v1_en.j2 +prompt_variables: + input_name: thread + unit_name: post/comment + +system_prompt_template: sequential_sentence_classification_system_v1_en.j2 +system_prompt_variables: + annotation_target: posts and comments of reddit threads + project_name: Discourse Act Analysis of Reddit Discussions + project_details: analyzing the discourse acts of Reddit threads. Posts and comments are categorized by coarse discourse acts. + annotation_guidelines: + - >- + question - A comment with a question or a request seeking some form of feedback, help, or other kinds of responses. While the comment may contain a question mark, it is not required. For instance, it might be posed in the form of a statement but still soliciting a response. Also, not everything that has a question mark is automatically a QUESTION. For instance, rhetorical questions are not seeking a response. Relation: This comment might be the first in a thread and have no relation to another comment. Or, it could be a clarifying or followup QUESTION linking to any prior comment. + - >- + answer - A comment that is responding to a QUESTION by answering the question or fulfilling the request. There can be more than one ANSWER responding to a QUESTION. Relation: An ANSWER is always linked to a QUESTION. + - >- + announcement - A comment that is presenting some new information to the community, such as a piece of news, a link to something, a story, an opinion, a review, or insight. Relation: This comment has no relation to a prior comment and is always the initial post in a thread. + - >- + agreement - A comment that is expressing agreement with some information presented in a prior comment. It can be agreeing with a point made, providing supporting evidence, providing a positive example or experience, or confirming or acknowledging a point made. Relation: This comment is always linked to a prior comment to which it is agreeing. + - >- + appreciation - A comment that is expressing thanks, appreciation, excitement, or praise in response to another comment. In contrast to AGREEMENT, it is not evaluating the merits of the points brought up. Comments of this category are more interpersonal as opposed to informational. Relation: This comment is always linked to a prior comment for which it is expressing appreciation. + - >- + disagreement - A comment that is correcting, criticizing, contradicting, or objecting to a point made in a prior comment. It can also be providing evidence to support its disagreement, such as an example or contrary anecdote.Relation: This comment is always linked to a prior comment to which it is disagreeing. + - >- + negative reaction - A comment that is expressing a negative reaction to a previous comment, such as attacking or mocking the commenter, or expressing emotions like disgust, derision, or anger, to the contents of the prior comment. This comment is not discussing the merits of the points made in a prior comment or trying to correct them. Relation: This comment is always linked to a prior comment to which it is negatively reacting. + - >- + elaboration - A comment that is adding additional information on to another comment. Oftentimes, one can imagine it simply appended to the end of the comment it elaborates on. One can elaborate on many kinds of comments, for instance, a questionasker elaborating on their question to provide more context, or someone elaborating on an answer to add more information. Relation: This comment is always linked to a prior comment upon which it is elaborating. + - >- + humor - This comment is primarily a joke, a piece of sarcasm, or a pun intended to get a laugh or be silly but not trying to add information. If a comment is sarcastic but using sarcasm to make a point or provide feedback, then it may belong in a different category. Relation: At times, this comment links to another comment but other times it may not be responding to anything. + - >- + other - A comment that does not fit any of the previous definitions. + +answer_schema: sequential_sentence_classification_schema.CoarseDiscourseSequentialSentenceClassificationSchemaV1 + +metrics: + - sequential_sentence_classification_metrics + +artifacts: + - sequential_sentence_classification_report + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/sequential_sentence_classification_csabstruct_v1.yaml b/benchmarks/configs/experiment/sequential_sentence_classification_csabstruct_v1.yaml new file mode 100644 index 000000000..1e6d4469f --- /dev/null +++ b/benchmarks/configs/experiment/sequential_sentence_classification_csabstruct_v1.yaml @@ -0,0 +1,40 @@ +defaults: + - /model: llama32_3b + - /dataset: csabstruct + +experiment_name: sequential_sentence_classification +# run_name: llama32_3b_csabstruct_zeroshot_v1 + +max_examples: 500 + +prompt_template: sequential_sentence_classification_v1_en.j2 +prompt_variables: + input_name: document + unit_name: sentence + +system_prompt_template: sequential_sentence_classification_system_v1_en.j2 +system_prompt_variables: + annotation_target: sentences of computer science abstracts + project_name: Rethorical Roles Analysis of CS Abstracts + project_details: analyzing the rhetorical roles of sentences in computer science abstracts. The abstracts are collected from the Semantic Scholar corpus. + annotation_guidelines: + - >- + background - Provides context or previous knowledge relevant to the research topic. Think of it as setting the stage for the study. + - >- + method - Describes the procedures and techniques used in the research. This includes the study design, data collection, and analysis methods. + - >- + objective - States the main goal or purpose of the research. What question is this work trying to answer? + - >- + other - Any sentence that does not fit into the above categories. This could be discussion, analysis, limitations, or concluding remarks. + - >- + result - Presents the findings or outcomes of the research. This often includes statistical data, tables, and figures. + +answer_schema: sequential_sentence_classification_schema.CSABStructSequentialSentenceClassificationSchemaV1 + +metrics: + - sequential_sentence_classification_metrics + +artifacts: + - sequential_sentence_classification_report + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/sequential_sentence_classification_daily_dialog_v1.yaml b/benchmarks/configs/experiment/sequential_sentence_classification_daily_dialog_v1.yaml new file mode 100644 index 000000000..6b23e2ebe --- /dev/null +++ b/benchmarks/configs/experiment/sequential_sentence_classification_daily_dialog_v1.yaml @@ -0,0 +1,46 @@ +defaults: + - /model: llama32_3b + - /dataset: daily-dialog + +experiment_name: sequential_sentence_classification +# run_name: llama32_3b_daily_dialog_zeroshot_v1 + +max_examples: 500 + +prompt_template: sequential_sentence_classification_v1_en.j2 +prompt_variables: + input_name: dialog + unit_name: utterance + +system_prompt_template: sequential_sentence_classification_system_v1_en.j2 +system_prompt_variables: + annotation_target: utterances in written dialogs + project_name: Emotion Analysis of Daily Dialogs + project_details: analyzing basic emotions of written dialogs on common topics. The dialogs reflect our daily communication way. + annotation_guidelines: + - >- + fear - A feeling of apprehension or dread in response to a perceived threat or danger. It can range from mild anxiety to intense terror. + - >- + disgust - A feeling of revulsion or aversion, often triggered by something perceived as unpleasant, unsanitary, or morally offensive. + - >- + neutral - A state of emotional balance or equilibrium, where no particular emotion is dominant. + - >- + excited - A state of heightened arousal and positive anticipation. It often involves feelings of enthusiasm, eagerness, and energy. + - >- + anger - A feeling of intense displeasure or hostility, often triggered by a perceived wrong or injustice. It can manifest as irritation, frustration, rage, or fury. + - >- + surprise - A brief emotional state in response to an unexpected event. It can be positive, negative, or neutral, depending on the nature of the surprise. + - >- + sadness - A feeling of sorrow, grief, or disappointment. It can range from mild melancholy to intense despair. + - >- + joy - A feeling of happiness, contentment, or pleasure. It can manifest as amusement or love. + +answer_schema: sequential_sentence_classification_schema.DailyDialogSequentialSentenceClassificationSchemaV1 + +metrics: + - sequential_sentence_classification_metrics + +artifacts: + - sequential_sentence_classification_report + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/sequential_sentence_classification_emotion_lines_v1.yaml b/benchmarks/configs/experiment/sequential_sentence_classification_emotion_lines_v1.yaml new file mode 100644 index 000000000..49ca6b426 --- /dev/null +++ b/benchmarks/configs/experiment/sequential_sentence_classification_emotion_lines_v1.yaml @@ -0,0 +1,46 @@ +defaults: + - /model: llama32_3b + - /dataset: emotion-lines + +experiment_name: sequential_sentence_classification +# run_name: llama32_3b_emotion_lines_zeroshot_v1 + +max_examples: 500 + +prompt_template: sequential_sentence_classification_v1_en.j2 +prompt_variables: + input_name: dialog + unit_name: utterance + +system_prompt_template: sequential_sentence_classification_system_v1_en.j2 +system_prompt_variables: + annotation_target: utterances in spoken dialogs + project_name: Emotion Analysis of Friends TV Show Dialogs + project_details: analyzing basic emotions of spoken dialogs. The dialogs are collected from Friends TV scripts. + annotation_guidelines: + - >- + fear - A feeling of apprehension or dread in response to a perceived threat or danger. It can range from mild anxiety to intense terror. + - >- + disgust - A feeling of revulsion or aversion, often triggered by something perceived as unpleasant, unsanitary, or morally offensive. + - >- + excited - A state of heightened arousal and positive anticipation. It often involves feelings of enthusiasm, eagerness, and energy. + - >- + anger - A feeling of intense displeasure or hostility, often triggered by a perceived wrong or injustice. It can manifest as irritation, frustration, rage, or fury. + - >- + surprise - A brief emotional state in response to an unexpected event. It can be positive, negative, or neutral, depending on the nature of the surprise. + - >- + sadness - A feeling of sorrow, grief, or disappointment. It can range from mild melancholy to intense despair. + - >- + joy - A feeling of happiness, contentment, or pleasure. It can manifest as amusement or love. + - >- + neutral - A state of emotional balance or equilibrium, where no particular emotion is dominant. + +answer_schema: sequential_sentence_classification_schema.EmotionLinesSequentialSentenceClassificationSchemaV1 + +metrics: + - sequential_sentence_classification_metrics + +artifacts: + - sequential_sentence_classification_report + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/sequential_sentence_classification_pubmed200k_v1.yaml b/benchmarks/configs/experiment/sequential_sentence_classification_pubmed200k_v1.yaml new file mode 100644 index 000000000..7bfae5da4 --- /dev/null +++ b/benchmarks/configs/experiment/sequential_sentence_classification_pubmed200k_v1.yaml @@ -0,0 +1,40 @@ +defaults: + - /model: llama32_3b + - /dataset: pubmed200k + +experiment_name: sequential_sentence_classification +# run_name: llama32_3b_pubmed200k_zeroshot_v1 + +max_examples: 500 + +prompt_template: sequential_sentence_classification_v1_en.j2 +prompt_variables: + input_name: document + unit_name: sentence + +system_prompt_template: sequential_sentence_classification_system_v1_en.j2 +system_prompt_variables: + annotation_target: sentences of PubMed abstracts + project_name: Rethorical Roles Analysis of PubMed Abstracts + project_details: analyzing the rhetorical roles of sentences in abstracts of randomized controlled trials. The abstracts are collected from PubMed. + annotation_guidelines: + - >- + background - Provides context or previous knowledge relevant to the research topic. Think of it as setting the stage for the study. + - >- + methods - Describes the procedures and techniques used in the research. This includes the study design, data collection, and analysis methods. + - >- + objective - States the main goal or purpose of the research. What question is this work trying to answer? + - >- + results - Presents the findings or outcomes of the research. This often includes statistical data, tables, and figures. + - >- + conclusions - Summarizes the key findings of the research and draw inferences from those findings. They provide closure to the abstract, summarizing the overall contribution of the research. + +answer_schema: sequential_sentence_classification_schema.Pubmed200KSequentialSentenceClassificationSchemaV1 + +metrics: + - sequential_sentence_classification_metrics + +artifacts: + - sequential_sentence_classification_report + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/span_classification_direct_quotation_v1.yaml b/benchmarks/configs/experiment/span_classification_direct_quotation_v1.yaml new file mode 100644 index 000000000..76ed8415f --- /dev/null +++ b/benchmarks/configs/experiment/span_classification_direct_quotation_v1.yaml @@ -0,0 +1,28 @@ +defaults: + - /model: llama32_3b + - /dataset: german-quotations-direct + +experiment_name: span_classification +# run_name: llama32_3b_direct_quotation_v1 + +max_examples: 10000 +sample_randomly: true +sample_random_state: 42 + +prompt_template: span_classification_v1_de.j2 +prompt_variables: + examples: | + {"category": "Sprecher", "text": "Angela Merkel"} + {"category": "Direkte Rede", "text": "\"Wir schaffen das!\""} + +system_prompt_template: span_classification_system_de.j2 + +answer_schema: span_classification_schema.DirectQuotationSpanClassificationSchemaV1 + +metrics: + - span_classification_metrics + +artifacts: + - span_classification_report + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/span_classification_fewnerd_coarse_v1.yaml b/benchmarks/configs/experiment/span_classification_fewnerd_coarse_v1.yaml new file mode 100644 index 000000000..b55ef34f4 --- /dev/null +++ b/benchmarks/configs/experiment/span_classification_fewnerd_coarse_v1.yaml @@ -0,0 +1,28 @@ +defaults: + - /model: llama32_3b + - /dataset: fewnerd-coarse + +experiment_name: span_classification +# run_name: llama32_3b_fewnerd_coarse_v1 + +max_examples: 100 +sample_randomly: true +sample_random_state: 42 + +prompt_template: span_classification_v1_en.j2 +prompt_variables: + examples: | + {"category": "art", "text": "Mona Lisa"} + {"category": "building", "text": "Eiffel Tower"} + +system_prompt_template: span_classification_system_en.j2 + +answer_schema: span_classification_schema.FewnerdCoarseSpanClassificationSchemaV1 + +metrics: + - span_classification_metrics + +artifacts: + - span_classification_report + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/span_classification_fewnerd_fine_v1.yaml b/benchmarks/configs/experiment/span_classification_fewnerd_fine_v1.yaml new file mode 100644 index 000000000..d6ee9f428 --- /dev/null +++ b/benchmarks/configs/experiment/span_classification_fewnerd_fine_v1.yaml @@ -0,0 +1,28 @@ +defaults: + - /model: llama32_3b + - /dataset: fewnerd-fine + +experiment_name: span_classification +# run_name: llama32_3b_fewnerd_fine_v1 + +max_examples: 10000 +sample_randomly: true +sample_random_state: 42 + +prompt_template: span_classification_v1_en.j2 +prompt_variables: + examples: | + {"category": "art - painting", "text": "Mona Lisa"} + {"category": "building - other", "text": "Eiffel Tower"} + +system_prompt_template: span_classification_system_en.j2 + +answer_schema: span_classification_schema.FewnerdFineSpanClassificationSchemaV1 + +metrics: + - span_classification_metrics + +artifacts: + - span_classification_report + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/span_classification_germanler_coarse_v1.yaml b/benchmarks/configs/experiment/span_classification_germanler_coarse_v1.yaml new file mode 100644 index 000000000..8ff9b1a1e --- /dev/null +++ b/benchmarks/configs/experiment/span_classification_germanler_coarse_v1.yaml @@ -0,0 +1,28 @@ +defaults: + - /model: llama32_3b + - /dataset: german-ler-coarse + +experiment_name: span_classification +# run_name: llama32_3b_germanler_coarse_v1 + +max_examples: 10000 +sample_randomly: true +sample_random_state: 42 + +prompt_template: span_classification_v1_de.j2 +prompt_variables: + examples: | + {"category": "person", "text": "Angela Merkel"} + {"category": "gesetz", "text": "Artikel 5"} + +system_prompt_template: span_classification_system_de.j2 + +answer_schema: span_classification_schema.GermanLERCoarseSpanClassificationSchemaV1 + +metrics: + - span_classification_metrics + +artifacts: + - span_classification_report + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/span_classification_germanler_fine_v1.yaml b/benchmarks/configs/experiment/span_classification_germanler_fine_v1.yaml new file mode 100644 index 000000000..73d863805 --- /dev/null +++ b/benchmarks/configs/experiment/span_classification_germanler_fine_v1.yaml @@ -0,0 +1,28 @@ +defaults: + - /model: llama32_3b + - /dataset: german-ler-fine + +experiment_name: span_classification +# run_name: llama32_3b_germanler_fine_v1 + +max_examples: 10000 +sample_randomly: true +sample_random_state: 42 + +prompt_template: span_classification_v1_de.j2 +prompt_variables: + examples: | + {"category": "Person", "text": "Angela Merkel"} + {"category": "Gesetz", "text": "Artikel 5"} + +system_prompt_template: span_classification_system_de.j2 + +answer_schema: span_classification_schema.GermanLERFineSpanClassificationSchemaV1 + +metrics: + - span_classification_metrics + +artifacts: + - span_classification_report + +temperature: 0.0 diff --git a/benchmarks/configs/experiment/template_filling_muc4_v1.yaml b/benchmarks/configs/experiment/template_filling_muc4_v1.yaml new file mode 100644 index 000000000..46f0e4726 --- /dev/null +++ b/benchmarks/configs/experiment/template_filling_muc4_v1.yaml @@ -0,0 +1,17 @@ +defaults: + - /model: llama32_3b + - /dataset: muc4 + +experiment_name: template_filling +# run_name: llama32_3b_muc4_v1 + +prompt_template: template_filling_muc4_v1_en.j2 + +system_prompt_template: template_filling_system_en.j2 + +answer_schema: template_filling_schema.TemplateFillingMUC4AnswerSchemaV1 + +metrics: + - template_filling_muc4_metrics + +temperature: 0.0 diff --git a/benchmarks/configs/model/llama32_3b.yaml b/benchmarks/configs/model/llama32_3b.yaml new file mode 100644 index 000000000..13aff7e1e --- /dev/null +++ b/benchmarks/configs/model/llama32_3b.yaml @@ -0,0 +1,4 @@ +name: meta-llama/Llama-3.2-3B-Instruct +alias: llama32_3b +max_len: 8192 +gpu_memory_utilization: 0.5 diff --git a/benchmarks/datasets/20newsgroups/.gitignore b/benchmarks/datasets/20newsgroups/.gitignore new file mode 100644 index 000000000..cc24f6078 --- /dev/null +++ b/benchmarks/datasets/20newsgroups/.gitignore @@ -0,0 +1,4 @@ +* +!.gitignore +!README.md +!20newsgroups_dataset_creation.py diff --git a/benchmarks/datasets/20newsgroups/20newsgroups_dataset_creation.py b/benchmarks/datasets/20newsgroups/20newsgroups_dataset_creation.py new file mode 100644 index 000000000..b733f5feb --- /dev/null +++ b/benchmarks/datasets/20newsgroups/20newsgroups_dataset_creation.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import argparse +import re +from pathlib import Path +from typing import Any, Literal, cast + +import pandas as pd +from sklearn.datasets import fetch_20newsgroups + +TARGET_CATEGORIES = [ + "alt.atheism", + "comp.graphics", + "sci.space", + "talk.politics.mideast", +] + + +def _clean_text(text: str, max_chars: int) -> str: + normalized = re.sub(r"\s+", " ", text).strip() + if len(normalized) <= max_chars: + return normalized + return normalized[:max_chars].rstrip() + + +def preprocess_20newsgroups( + output_raw: Path, + output_processed: Path, + subset: Literal["train", "test", "all"], + samples_per_class: int, + seed: int, + max_chars: int, +) -> None: + dataset = cast( + Any, + fetch_20newsgroups( + subset=subset, + categories=TARGET_CATEGORIES, + remove=("headers", "footers", "quotes"), + ), + ) + + targets = [int(item) for item in dataset.target] + + df = pd.DataFrame( + { + "document_id": list(range(len(dataset.data))), + "document_text": [ + _clean_text(str(text), max_chars=max_chars) for text in dataset.data + ], + "target": targets, + } + ) + df["label"] = df["target"].map(lambda x: TARGET_CATEGORIES[int(x)]) + + # remove rows with empty document text after cleaning + df = df[df["document_text"].str.strip() != ""].copy() + + output_raw.parent.mkdir(parents=True, exist_ok=True) + output_processed.parent.mkdir(parents=True, exist_ok=True) + + df.to_parquet(output_raw, index=False) + + sampled_parts: list[pd.DataFrame] = [] + for _, group in df.groupby("label"): + sampled_parts.append( + group.sample( + n=min(samples_per_class, len(group)), + random_state=seed, + ) + ) + + sampled_df = pd.concat(sampled_parts, ignore_index=True) + + sampled_df.to_parquet(output_processed, index=False) + + print("20 Newsgroups preprocessing completed.") + print(f"Raw rows: {len(df)} -> {output_raw}") + print(f"Label distribution (raw):") + print(df["label"].value_counts().to_dict()) + print(f"Sampled rows: {len(sampled_df)} -> {output_processed}") + print("Label distribution (sampled):") + print(sampled_df["label"].value_counts().to_dict()) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Preprocess 20 Newsgroups for benchmarking" + ) + parser.add_argument("--subset", default="test", choices=["train", "test", "all"]) + parser.add_argument("--samples-per-class", type=int, default=30) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--max-chars", type=int, default=3000) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + project_root = Path(__file__).resolve().parents[2] + + preprocess_20newsgroups( + output_raw=project_root / "datasets/20newsgroups/test_full_raw.parquet", + output_processed=project_root / "datasets/20newsgroups/test_sampled.parquet", + subset=args.subset, + samples_per_class=args.samples_per_class, + seed=args.seed, + max_chars=args.max_chars, + ) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/datasets/20newsgroups/README.md b/benchmarks/datasets/20newsgroups/README.md new file mode 100644 index 000000000..67142e9fc --- /dev/null +++ b/benchmarks/datasets/20newsgroups/README.md @@ -0,0 +1,70 @@ +# 20 Newsgroups (Benchmark Dataset) + +## What Is This Dataset About? + +This dataset is a 4-class subset of the classic 20 Newsgroups corpus for topic/document classification. + +In this repository, we keep the following target classes: + +- `alt.atheism` +- `comp.graphics` +- `sci.space` +- `talk.politics.mideast` + +## Where Can It Be Found? + +- Scikit-learn dataset documentation: + - https://scikit-learn.org/stable/datasets/real_world.html#the-20-newsgroups-text-dataset +- Loaded in our pipeline via `sklearn.datasets.fetch_20newsgroups`. + +## Links (Website / Download / Citation) + +- Scikit-learn API reference: + - https://scikit-learn.org/stable/modules/generated/sklearn.datasets.fetch_20newsgroups.html + +## Benchmark Task Usage + +- Task 1: Document Classification + +## Dataset Size (Current Files) + +- `test_sampled.parquet` (main benchmark file): 120 samples +- `test_full_raw.parquet` (reference full split): 1478 samples + +## How We Preprocess It + +Preprocessing is implemented in `20newsgroups_dataset_creation.py`. + +Main steps: + +1. Load subset (`train`, `test`, or `all`) with selected categories. +2. Remove metadata noise from raw posts: + - headers + - footers + - quotes +3. Normalize whitespace and trim each document to a maximum character length (`max_chars`, default `3000`). +4. Build a dataframe with: + - `document_id` + - `document_text` + - numeric `target` + - string `label` mapped from `target` +5. Save full processed split to `test_full_raw.parquet`. +6. Create label-balanced sampled set (default up to `30` samples per class) and save to `test_sampled.parquet`. + +## Final Dataset Structure + +### File: `test_full_raw.parquet` + +- `document_id`: integer document identifier +- `document_text`: cleaned text content +- `target`: integer class index +- `label`: class name (string) + +### File: `test_sampled.parquet` + +- Same schema as `test_full_raw.parquet` +- Contains class-balanced sampled records + +## Notes + +- The script currently writes output paths named `test_full_raw.parquet` and `test_sampled.parquet` regardless of selected subset argument. diff --git a/benchmarks/datasets/bbc/.gitignore b/benchmarks/datasets/bbc/.gitignore new file mode 100644 index 000000000..63de53581 --- /dev/null +++ b/benchmarks/datasets/bbc/.gitignore @@ -0,0 +1,4 @@ +* +!.gitignore +!README.md +!bbc_dataset_creation.ipynb diff --git a/benchmarks/datasets/bbc/README.md b/benchmarks/datasets/bbc/README.md new file mode 100644 index 000000000..e2d5acd7e --- /dev/null +++ b/benchmarks/datasets/bbc/README.md @@ -0,0 +1,141 @@ +# BBC News Alltime (Benchmark Dataset) + +## What Is This Dataset About? + +A large-scale BBC news corpus used for topic/document classification. + +The raw data contains article metadata and article text. In our benchmark, we derive topic labels from article link/tag structure and create a cleaned classification-ready dataset. + +## Where Can It Be Found? + +- Hugging Face dataset: + - https://huggingface.co/datasets/RealTimeData/bbc_news_alltime + +## Links (Website / Download / Citation) + +- Dataset card: + - https://huggingface.co/datasets/RealTimeData/bbc_news_alltime + +## Benchmark Task Usage + +- Task 1: Document Classification + +## Dataset Size (Current Files) + +- `bbc_cleaned.parquet` (main benchmark file): 81182 samples + +## Label Space (Most Important) + +The classification labels are derived from URL tags. + +### `main_tag` classes (4) + +- `uk` (35916) +- `misc` (21268) +- `world` (18642) +- `sport` (5356) + +### `sub_tag` classes (26) + +- `africa` (918) +- `asia` (2776) +- `athletics` (182) +- `australia` (700) +- `boxing` (158) +- `business` (8026) +- `cricket` (500) +- `education` (1175) +- `election` (614) +- `england` (14475) +- `entertainment` (4776) +- `europe` (6332) +- `football` (3393) +- `formula1` (179) +- `health` (2897) +- `latin-america` (728) +- `middle-east` (1672) +- `northern-ireland` (3433) +- `politics` (7590) +- `rugby` (347) +- `science` (1817) +- `scotland` (5265) +- `technology` (1963) +- `tennis` (597) +- `us` (5516) +- `wales` (5153) + +### `tag` classes (26) + +- `uk/england` (14475) +- `misc/business` (8026) +- `uk/politics` (7590) +- `world/europe` (6332) +- `world/us` (5516) +- `uk/scotland` (5265) +- `uk/wales` (5153) +- `uk/entertainment` (4776) +- `uk/northern-ireland` (3433) +- `sport/football` (3393) +- `uk/health` (2897) +- `world/asia` (2776) +- `uk/technology` (1963) +- `uk/science` (1817) +- `world/middle-east` (1672) +- `uk/education` (1175) +- `world/africa` (918) +- `world/latin-america` (728) +- `world/australia` (700) +- `uk/election` (614) +- `sport/tennis` (597) +- `sport/cricket` (500) +- `sport/rugby` (347) +- `sport/athletics` (182) +- `sport/formula1` (179) +- `sport/boxing` (158) + +## How We Preprocess It + +Preprocessing is implemented in `bbc_dataset_creation.ipynb`. + +Main steps: + +1. Download/concatenate monthly slices (2018-2023) from Hugging Face. +2. Save concatenated raw dataset to `bbc_news_alltime.parquet`. +3. Drop noisy columns (`authors`, `top_image`). +4. Parse article `link` path into hierarchical tags. +5. Derive: + - `tags` + - `tags_len` + - `main_tag` + - `sub_tag` + - merged `tag` +6. Filter noisy/rare groups and normalize some sub-tag names. +7. Save cleaned dataset to `bbc_cleaned.parquet`. + +## Final Dataset Structure + +### File: `bbc_news_alltime.parquet` (raw combined) + +- `title` +- `published_date` +- `description` +- `section` +- `content` +- `link` +- `__index_level_0__` (pandas index artifact) + +### File: `bbc_cleaned.parquet` (classification-ready) + +- `title` +- `published_date` +- `description` +- `section` +- `content` +- `link` +- `count` +- `tags` +- `tags_len` +- `main_tag` +- `sub_tag` +- `tag` +- `__index_level_0__` (pandas index artifact) diff --git a/benchmarks/datasets/bbc/bbc_dataset_creation.ipynb b/benchmarks/datasets/bbc/bbc_dataset_creation.ipynb new file mode 100644 index 000000000..fddf49477 --- /dev/null +++ b/benchmarks/datasets/bbc/bbc_dataset_creation.ipynb @@ -0,0 +1,1390 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import datasets\n", + "import pandas as pd" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Dataset Download" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dfs = []\n", + "# iterate over 2018 - 2023\n", + "for year in range(2018, 2024):\n", + " # iteratoe over months 1 - 12\n", + " for month in range(1, 13):\n", + " # create YYYY-MM string\n", + " date = f\"{year}-{month:02d}\"\n", + "\n", + " # get the dataset\n", + " ds = datasets.load_dataset('RealTimeData/bbc_news_alltime', date)\n", + " try:\n", + " df = ds[\"train\"].to_pandas()\n", + " except:\n", + " continue\n", + "\n", + " # append the data to the list\n", + " dfs.append(df)\n", + "\n", + "# concatenate all the dataframes\n", + "df = pd.concat(dfs)\n", + "\n", + "# remove the authors and top_image columns\n", + "df = df.drop(columns=[\"authors\", \"top_image\"])\n", + "\n", + "# save the data to a new parquet file\n", + "df.to_parquet(\"bbc_news_alltime.parquet\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Dataset Processing" + ] + }, + { + "cell_type": "code", + "execution_count": 139, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.read_parquet(\"datasets/bbc/bbc_news_alltime.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 140, + "metadata": {}, + "outputs": [], + "source": [ + "# count slashes\n", + "df[\"count\"] = df[\"link\"].apply(lambda x: len(x.split(\"/\")))" + ] + }, + { + "cell_type": "code", + "execution_count": 141, + "metadata": {}, + "outputs": [], + "source": [ + "# filter out all rows where the count is not equal to 5 or 6\n", + "df = df[(df[\"count\"] == 5) | (df[\"count\"] == 6)]" + ] + }, + { + "cell_type": "code", + "execution_count": 145, + "metadata": {}, + "outputs": [], + "source": [ + "from typing import List\n", + "\n", + "\n", + "def extract_tags(link: str) -> List[str]:\n", + " tags: List[str] = []\n", + " components = link.split(\"/\")\n", + " if len(components) == 6:\n", + " tags.append(components[3])\n", + " tags.append(components[4])\n", + " if len(components) == 5:\n", + " tags.append(components[3])\n", + "\n", + " article_name = components[-1]\n", + " tags.extend(article_name.split(\"-\")[:-1])\n", + " return tags\n", + "\n", + "\n", + "def extract_main_tags(link: str) -> List[str]:\n", + " tags = extract_tags(link)\n", + " if len(tags) > 1:\n", + " if tags[0] == \"news\" and tags[1] == \"world\":\n", + " return tags[1:]\n", + " if tags[0] == \"news\" and tags[1] == \"uk\":\n", + " return tags[1:]\n", + " if tags[0] == \"news\":\n", + " return [\"misc\"] + tags[1:]\n", + " \n", + " return tags\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 146, + "metadata": {}, + "outputs": [], + "source": [ + "# extract tags from links\n", + "df[\"tags\"] = df[\"link\"].apply(lambda x: extract_main_tags(x))" + ] + }, + { + "cell_type": "code", + "execution_count": 147, + "metadata": {}, + "outputs": [], + "source": [ + "# count tags\n", + "df[\"tags_len\"] = df[\"tags\"].apply(lambda x: len(x))" + ] + }, + { + "cell_type": "code", + "execution_count": 150, + "metadata": {}, + "outputs": [], + "source": [ + "# remove colums with tags_len <= 1\n", + "df = df[df[\"tags_len\"] > 1]" + ] + }, + { + "cell_type": "code", + "execution_count": 152, + "metadata": {}, + "outputs": [], + "source": [ + "df[\"main_tag\"] = df[\"tags\"].apply(lambda x: x[0])\n", + "df[\"sub_tag\"] = df[\"tags\"].apply(lambda x: x[1])" + ] + }, + { + "cell_type": "code", + "execution_count": 153, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
titlepublished_datedescriptionsectioncontentlinkcounttagstags_lenmain_tagsub_tag
59Jake Livermore: West Brom say West Ham fan alt...2018-01-03West Brom say Jake Livermore confronted a West...NoneWest Brom say Jake Livermore confronted a West...http://www.bbc.co.uk/sport/football/425583606[sport, football]2sportfootball
61Coronation Street's first barmaid Doreen Keogh...2018-01-03The Irish actress played Rovers Return's first...Entertainment & ArtsActress Doreen Keogh, who played the first Cor...http://www.bbc.co.uk/news/entertainment-arts-4...5[misc, entertainment, arts]3miscentertainment
62A&E doctor: 'I feel like I'm fighting a losing...2018-01-03Tens of thousands of non-urgent NHS operations...NoneTens of thousands of non-urgent NHS operations...http://www.bbc.co.uk/news/health-425535975[misc, health]2mischealth
63AI early diagnosis could save heart and cancer...2018-01-03The systems will save billions of pounds by en...HealthSir John Bell believes that artificial intelli...http://www.bbc.co.uk/news/health-423572575[misc, health]2mischealth
64Triple killer Theodore Johnson admits murderin...2018-01-03Theodore Johnson already had convictions for k...LondonTheodore Johnson beat Angela Best with a claw ...http://www.bbc.co.uk/news/uk-england-london-42...5[uk, england, london]3ukengland
65Iuliana Tudos Finsbury Park death: Man charged...2018-01-03Kasim Lewis, 31, is charged with murder after ...LondonThe body of Iuliana Tudos was discovered near ...http://www.bbc.co.uk/news/uk-england-london-42...5[uk, england, london]3ukengland
66Berlin reels after nine inmates escape Plötzen...2018-01-03Plötzensee jail \"lost\" the prisoners in four i...EuropePrisoners can be seen escaping from a shaft in...http://www.bbc.co.uk/news/world-europe-425517325[world, europe]2worldeurope
67Manchester City 3-1 Watford - BBC Sport2018-01-03Runaway leaders Manchester City show no ill-ef...NoneLast updated on .From the section Premier Leag...http://www.bbc.co.uk/sport/football/424110776[sport, football]2sportfootball
70Hospital apologises after baby's skull cut in ...2018-01-03The baby died three hours after suffering a sc...NottinghamCarson's mother Claire Smith, pictured with hi...http://www.bbc.co.uk/news/uk-england-nottingha...5[uk, england, nottinghamshire]3ukengland
71Freed hostage Joshua Boyle faces 15 criminal c...2018-01-03Canadian Joshua Boyle and his family were held...US & CanadaA Canadian man who was held hostage for five y...http://www.bbc.co.uk/news/world-us-canada-4254...5[world, us, canada]3worldus
72Royal wedding: Windsor council leader in beggi...2018-01-03Windsor has an \"epidemic of rough sleeping and...BerkshireThis video can not be played To play this vide...http://www.bbc.co.uk/news/uk-england-berkshire...5[uk, england, berkshire]3ukengland
73Major flaw in millions of Intel chips revealed...2018-01-03A serious bug will need to be patched in milli...TechnologyA serious flaw in the design of Intel's chips ...http://www.bbc.co.uk/news/technology-425538185[misc, technology]2misctechnology
74Andargachew Tsege's wife hopeful Ethiopia will...2018-01-03The wife of a man being held in Ethiopia is \"o...UKAndy Tsege has not seen his wife or three chil...http://www.bbc.co.uk/news/uk-england-425590685[uk, england]2ukengland
75Hospitals to delay non-urgent operations - BBC...2018-01-03NHS England says tens of thousands of non-urge...HealthTens of thousands of non-urgent NHS operations...http://www.bbc.co.uk/news/health-425414125[misc, health]2mischealth
76Beauty and the Beast pips Star Wars to be the ...2018-01-03Emma Watson's Beauty and the Beast beats The L...Entertainment & ArtsNew character Rose was introduced in The Last ...http://www.bbc.co.uk/news/entertainment-arts-4...5[misc, entertainment, arts]3miscentertainment
77Each of us streamed 1,036 songs last year (and...2018-01-03UK music fans streamed 68.1 billion songs last...Entertainment & ArtsEd Sheeran, Stormzy and Dua Lipa had some of t...http://www.bbc.co.uk/news/entertainment-arts-4...5[misc, entertainment, arts]3miscentertainment
79Dundalk stabbing suspect may have had 'two kni...2018-01-03One man is dead and two others are injured in ...EuropeThis video can not be played To play this vide...http://www.bbc.co.uk/news/world-europe-425541275[world, europe]2worldeurope
81Christmas sales surprise for Next - BBC News2018-01-03Next shares jump after after cold weather boos...BusinessSales at Next have unexpectedly risen over the...http://www.bbc.co.uk/news/business-425509175[misc, business]2miscbusiness
82A&E: People on beds 'as far as you could see' ...2018-01-03For four hours, Yvonne Beer, 87, was tied in a...Hereford & WorcesterYvonne Beer is 87 and suffers from dementia\\n\\...http://www.bbc.co.uk/news/uk-england-hereford-...5[uk, england, hereford, worcester]4ukengland
84Trump Bannon row: 11 explosive claims from new...2018-01-03Donald Trump was \"befuddled\" by his election w...US & CanadaMr Trump sits at the White House Resolute Desk...http://www.bbc.co.uk/news/world-us-canada-4255...5[world, us, canada]3worldus
85Six held in far-right group National Action in...2018-01-03West Midlands Police say the six are being hel...EnglandThis video can not be played To play this vide...http://www.bbc.co.uk/news/uk-england-425527505[uk, england]2ukengland
87YouTube singer Chrissy Chambers wins revenge p...2018-01-17Chrissy Chambers is awarded damages from her e...TechnologyChrissy Chambers, seen on the left, makes YouT...http://www.bbc.co.uk/news/technology-427208695[misc, technology]2misctechnology
88'Racist' H&M coolest monkey hoodie banned by e...2018-01-17Ebay says it will \"remove items listed malicio...NewsbeateBay has banned people from selling the \"racis...http://www.bbc.co.uk/newsbeat/articles/427179336[newsbeat, articles]2newsbeatarticles
89Meteor lights up Michigan skies - BBC News2018-01-17The US Geological Survey said seismic waves pr...NoneFootage of a flash of light appearing across M...http://www.bbc.co.uk/news/science-environment-...5[misc, science, environment]3miscscience
90Nottingham station fire: Sprinklers 'could hav...2018-01-17Sprinklers could have prevented the Nottingham...NottinghamThis video can not be played To play this vide...http://www.bbc.co.uk/news/uk-england-nottingha...5[uk, england, nottinghamshire]3ukengland
91Bristol Museum sold unauthorised Banksy angel ...2018-01-17Bristol Museum and Art Gallery sold thousands ...BristolA museum in Banksy's home city has been caught...http://www.bbc.co.uk/news/uk-england-bristol-4...5[uk, england, bristol]3ukengland
92Most new cars must be electric by 2030, minist...2018-01-17Three-fifths of new cars must be electric by 2...Science & EnvironmentThis video can not be played To play this vide...http://www.bbc.co.uk/news/science-environment-...5[misc, science, environment]3miscscience
93Carillion apprentices among casualties as firm...2018-01-17The future of about 1,400 apprentices at train...Family & EducationCarillion apprentice Kyle Fitzsimmons is weeks...http://www.bbc.co.uk/news/education-427095125[misc, education]2misceducation
94Tesco delays Clubcard rewards cut after backla...2018-01-17The supermarket announces a grace period until...BusinessTesco is delaying a cut to its biggest Clubcar...http://www.bbc.co.uk/news/business-427151435[misc, business]2miscbusiness
95YouTube toughens advert payment rules - BBC News2018-01-17The video-clip platform will require creators ...TechnologyYouTube creators will need to have more than 1...http://www.bbc.co.uk/news/technology-427163935[misc, technology]2misctechnology
96Margaret Atwood faces feminist backlash for #M...2018-01-17The Handmaid's Tale author said she was depict...US & CanadaAuthor Margaret Atwood has sparked a Twitter s...http://www.bbc.co.uk/news/world-us-canada-4270...5[world, us, canada]3worldus
97Winter Olympics: How good is North Korea at sp...2018-01-17Reality Check looks at the sporting skills of ...AsiaRi Se-gwang winning gold for North Korea in th...http://www.bbc.co.uk/news/world-asia-pacific-4...5[world, asia, pacific]3worldasia
98Dozens of women describe abuse by ex-doctor La...2018-01-17Dozens of women abused by ex-Team USA gymnasti...NoneNearly 100 women are expected to testify durin...http://www.bbc.co.uk/news/world-us-canada-4271...5[world, us, canada]3worldus
99Celine Dookhran trial: Builder 'kidnapped and ...2018-01-17A builder planned for weeks to abduct two wome...LondonCeline Dookhran was found dead in an empty hou...http://www.bbc.co.uk/news/uk-england-london-42...5[uk, england, london]3ukengland
100Dylan Farrow: Outrage after 'years of being ig...2018-01-17The director's adopted daughter discusses assa...Entertainment & ArtsThis video can not be played To play this vide...http://www.bbc.co.uk/news/entertainment-arts-4...5[misc, entertainment, arts]3miscentertainment
101Liverpool footballer Jon Flanagan sentenced fo...2018-01-17Liverpool FC defender Jon Flanagan \"slammed\" h...LiverpoolLiverpool footballer Jon Flanagan has been sen...http://www.bbc.co.uk/news/uk-england-merseysid...5[uk, england, merseyside]3ukengland
102Bitcoin dips below $10,000 for first time sinc...2018-01-17The crypto-currency has nearly halved in value...TechnologyBitcoin came close to crossing $20,000 in Dece...http://www.bbc.co.uk/news/technology-427176395[misc, technology]2misctechnology
103Man's bomb hoax to stop wife’s night out in Mi...2018-01-17Court hears Mo Ahmed was angry his wife was go...EnglandMo Ahmed was given a suspended sentence at Exe...http://www.bbc.co.uk/news/uk-england-427235465[uk, england]2ukengland
104Tory MP Ben Bradley 'sorry' for blog post abou...2018-01-17In 2012 Ben Bradley's blog post criticised wha...NottinghamBen Bradley became the MP for Mansfield in Jun...http://www.bbc.co.uk/news/uk-politics-427121805[uk, politics]2ukpolitics
105Ealing Council paves way to ban anti-abortion ...2018-01-17Council cabinet members vote in favour of bann...LondonThis video can not be played To play this vide...http://www.bbc.co.uk/news/uk-england-london-42...5[uk, england, london]3ukengland
\n", + "
" + ], + "text/plain": [ + " title published_date \\\n", + "59 Jake Livermore: West Brom say West Ham fan alt... 2018-01-03 \n", + "61 Coronation Street's first barmaid Doreen Keogh... 2018-01-03 \n", + "62 A&E doctor: 'I feel like I'm fighting a losing... 2018-01-03 \n", + "63 AI early diagnosis could save heart and cancer... 2018-01-03 \n", + "64 Triple killer Theodore Johnson admits murderin... 2018-01-03 \n", + "65 Iuliana Tudos Finsbury Park death: Man charged... 2018-01-03 \n", + "66 Berlin reels after nine inmates escape Plötzen... 2018-01-03 \n", + "67 Manchester City 3-1 Watford - BBC Sport 2018-01-03 \n", + "70 Hospital apologises after baby's skull cut in ... 2018-01-03 \n", + "71 Freed hostage Joshua Boyle faces 15 criminal c... 2018-01-03 \n", + "72 Royal wedding: Windsor council leader in beggi... 2018-01-03 \n", + "73 Major flaw in millions of Intel chips revealed... 2018-01-03 \n", + "74 Andargachew Tsege's wife hopeful Ethiopia will... 2018-01-03 \n", + "75 Hospitals to delay non-urgent operations - BBC... 2018-01-03 \n", + "76 Beauty and the Beast pips Star Wars to be the ... 2018-01-03 \n", + "77 Each of us streamed 1,036 songs last year (and... 2018-01-03 \n", + "79 Dundalk stabbing suspect may have had 'two kni... 2018-01-03 \n", + "81 Christmas sales surprise for Next - BBC News 2018-01-03 \n", + "82 A&E: People on beds 'as far as you could see' ... 2018-01-03 \n", + "84 Trump Bannon row: 11 explosive claims from new... 2018-01-03 \n", + "85 Six held in far-right group National Action in... 2018-01-03 \n", + "87 YouTube singer Chrissy Chambers wins revenge p... 2018-01-17 \n", + "88 'Racist' H&M coolest monkey hoodie banned by e... 2018-01-17 \n", + "89 Meteor lights up Michigan skies - BBC News 2018-01-17 \n", + "90 Nottingham station fire: Sprinklers 'could hav... 2018-01-17 \n", + "91 Bristol Museum sold unauthorised Banksy angel ... 2018-01-17 \n", + "92 Most new cars must be electric by 2030, minist... 2018-01-17 \n", + "93 Carillion apprentices among casualties as firm... 2018-01-17 \n", + "94 Tesco delays Clubcard rewards cut after backla... 2018-01-17 \n", + "95 YouTube toughens advert payment rules - BBC News 2018-01-17 \n", + "96 Margaret Atwood faces feminist backlash for #M... 2018-01-17 \n", + "97 Winter Olympics: How good is North Korea at sp... 2018-01-17 \n", + "98 Dozens of women describe abuse by ex-doctor La... 2018-01-17 \n", + "99 Celine Dookhran trial: Builder 'kidnapped and ... 2018-01-17 \n", + "100 Dylan Farrow: Outrage after 'years of being ig... 2018-01-17 \n", + "101 Liverpool footballer Jon Flanagan sentenced fo... 2018-01-17 \n", + "102 Bitcoin dips below $10,000 for first time sinc... 2018-01-17 \n", + "103 Man's bomb hoax to stop wife’s night out in Mi... 2018-01-17 \n", + "104 Tory MP Ben Bradley 'sorry' for blog post abou... 2018-01-17 \n", + "105 Ealing Council paves way to ban anti-abortion ... 2018-01-17 \n", + "\n", + " description section \\\n", + "59 West Brom say Jake Livermore confronted a West... None \n", + "61 The Irish actress played Rovers Return's first... Entertainment & Arts \n", + "62 Tens of thousands of non-urgent NHS operations... None \n", + "63 The systems will save billions of pounds by en... Health \n", + "64 Theodore Johnson already had convictions for k... London \n", + "65 Kasim Lewis, 31, is charged with murder after ... London \n", + "66 Plötzensee jail \"lost\" the prisoners in four i... Europe \n", + "67 Runaway leaders Manchester City show no ill-ef... None \n", + "70 The baby died three hours after suffering a sc... Nottingham \n", + "71 Canadian Joshua Boyle and his family were held... US & Canada \n", + "72 Windsor has an \"epidemic of rough sleeping and... Berkshire \n", + "73 A serious bug will need to be patched in milli... Technology \n", + "74 The wife of a man being held in Ethiopia is \"o... UK \n", + "75 NHS England says tens of thousands of non-urge... Health \n", + "76 Emma Watson's Beauty and the Beast beats The L... Entertainment & Arts \n", + "77 UK music fans streamed 68.1 billion songs last... Entertainment & Arts \n", + "79 One man is dead and two others are injured in ... Europe \n", + "81 Next shares jump after after cold weather boos... Business \n", + "82 For four hours, Yvonne Beer, 87, was tied in a... Hereford & Worcester \n", + "84 Donald Trump was \"befuddled\" by his election w... US & Canada \n", + "85 West Midlands Police say the six are being hel... England \n", + "87 Chrissy Chambers is awarded damages from her e... Technology \n", + "88 Ebay says it will \"remove items listed malicio... Newsbeat \n", + "89 The US Geological Survey said seismic waves pr... None \n", + "90 Sprinklers could have prevented the Nottingham... Nottingham \n", + "91 Bristol Museum and Art Gallery sold thousands ... Bristol \n", + "92 Three-fifths of new cars must be electric by 2... Science & Environment \n", + "93 The future of about 1,400 apprentices at train... Family & Education \n", + "94 The supermarket announces a grace period until... Business \n", + "95 The video-clip platform will require creators ... Technology \n", + "96 The Handmaid's Tale author said she was depict... US & Canada \n", + "97 Reality Check looks at the sporting skills of ... Asia \n", + "98 Dozens of women abused by ex-Team USA gymnasti... None \n", + "99 A builder planned for weeks to abduct two wome... London \n", + "100 The director's adopted daughter discusses assa... Entertainment & Arts \n", + "101 Liverpool FC defender Jon Flanagan \"slammed\" h... Liverpool \n", + "102 The crypto-currency has nearly halved in value... Technology \n", + "103 Court hears Mo Ahmed was angry his wife was go... England \n", + "104 In 2012 Ben Bradley's blog post criticised wha... Nottingham \n", + "105 Council cabinet members vote in favour of bann... London \n", + "\n", + " content \\\n", + "59 West Brom say Jake Livermore confronted a West... \n", + "61 Actress Doreen Keogh, who played the first Cor... \n", + "62 Tens of thousands of non-urgent NHS operations... \n", + "63 Sir John Bell believes that artificial intelli... \n", + "64 Theodore Johnson beat Angela Best with a claw ... \n", + "65 The body of Iuliana Tudos was discovered near ... \n", + "66 Prisoners can be seen escaping from a shaft in... \n", + "67 Last updated on .From the section Premier Leag... \n", + "70 Carson's mother Claire Smith, pictured with hi... \n", + "71 A Canadian man who was held hostage for five y... \n", + "72 This video can not be played To play this vide... \n", + "73 A serious flaw in the design of Intel's chips ... \n", + "74 Andy Tsege has not seen his wife or three chil... \n", + "75 Tens of thousands of non-urgent NHS operations... \n", + "76 New character Rose was introduced in The Last ... \n", + "77 Ed Sheeran, Stormzy and Dua Lipa had some of t... \n", + "79 This video can not be played To play this vide... \n", + "81 Sales at Next have unexpectedly risen over the... \n", + "82 Yvonne Beer is 87 and suffers from dementia\\n\\... \n", + "84 Mr Trump sits at the White House Resolute Desk... \n", + "85 This video can not be played To play this vide... \n", + "87 Chrissy Chambers, seen on the left, makes YouT... \n", + "88 eBay has banned people from selling the \"racis... \n", + "89 Footage of a flash of light appearing across M... \n", + "90 This video can not be played To play this vide... \n", + "91 A museum in Banksy's home city has been caught... \n", + "92 This video can not be played To play this vide... \n", + "93 Carillion apprentice Kyle Fitzsimmons is weeks... \n", + "94 Tesco is delaying a cut to its biggest Clubcar... \n", + "95 YouTube creators will need to have more than 1... \n", + "96 Author Margaret Atwood has sparked a Twitter s... \n", + "97 Ri Se-gwang winning gold for North Korea in th... \n", + "98 Nearly 100 women are expected to testify durin... \n", + "99 Celine Dookhran was found dead in an empty hou... \n", + "100 This video can not be played To play this vide... \n", + "101 Liverpool footballer Jon Flanagan has been sen... \n", + "102 Bitcoin came close to crossing $20,000 in Dece... \n", + "103 Mo Ahmed was given a suspended sentence at Exe... \n", + "104 Ben Bradley became the MP for Mansfield in Jun... \n", + "105 This video can not be played To play this vide... \n", + "\n", + " link count \\\n", + "59 http://www.bbc.co.uk/sport/football/42558360 6 \n", + "61 http://www.bbc.co.uk/news/entertainment-arts-4... 5 \n", + "62 http://www.bbc.co.uk/news/health-42553597 5 \n", + "63 http://www.bbc.co.uk/news/health-42357257 5 \n", + "64 http://www.bbc.co.uk/news/uk-england-london-42... 5 \n", + "65 http://www.bbc.co.uk/news/uk-england-london-42... 5 \n", + "66 http://www.bbc.co.uk/news/world-europe-42551732 5 \n", + "67 http://www.bbc.co.uk/sport/football/42411077 6 \n", + "70 http://www.bbc.co.uk/news/uk-england-nottingha... 5 \n", + "71 http://www.bbc.co.uk/news/world-us-canada-4254... 5 \n", + "72 http://www.bbc.co.uk/news/uk-england-berkshire... 5 \n", + "73 http://www.bbc.co.uk/news/technology-42553818 5 \n", + "74 http://www.bbc.co.uk/news/uk-england-42559068 5 \n", + "75 http://www.bbc.co.uk/news/health-42541412 5 \n", + "76 http://www.bbc.co.uk/news/entertainment-arts-4... 5 \n", + "77 http://www.bbc.co.uk/news/entertainment-arts-4... 5 \n", + "79 http://www.bbc.co.uk/news/world-europe-42554127 5 \n", + "81 http://www.bbc.co.uk/news/business-42550917 5 \n", + "82 http://www.bbc.co.uk/news/uk-england-hereford-... 5 \n", + "84 http://www.bbc.co.uk/news/world-us-canada-4255... 5 \n", + "85 http://www.bbc.co.uk/news/uk-england-42552750 5 \n", + "87 http://www.bbc.co.uk/news/technology-42720869 5 \n", + "88 http://www.bbc.co.uk/newsbeat/articles/42717933 6 \n", + "89 http://www.bbc.co.uk/news/science-environment-... 5 \n", + "90 http://www.bbc.co.uk/news/uk-england-nottingha... 5 \n", + "91 http://www.bbc.co.uk/news/uk-england-bristol-4... 5 \n", + "92 http://www.bbc.co.uk/news/science-environment-... 5 \n", + "93 http://www.bbc.co.uk/news/education-42709512 5 \n", + "94 http://www.bbc.co.uk/news/business-42715143 5 \n", + "95 http://www.bbc.co.uk/news/technology-42716393 5 \n", + "96 http://www.bbc.co.uk/news/world-us-canada-4270... 5 \n", + "97 http://www.bbc.co.uk/news/world-asia-pacific-4... 5 \n", + "98 http://www.bbc.co.uk/news/world-us-canada-4271... 5 \n", + "99 http://www.bbc.co.uk/news/uk-england-london-42... 5 \n", + "100 http://www.bbc.co.uk/news/entertainment-arts-4... 5 \n", + "101 http://www.bbc.co.uk/news/uk-england-merseysid... 5 \n", + "102 http://www.bbc.co.uk/news/technology-42717639 5 \n", + "103 http://www.bbc.co.uk/news/uk-england-42723546 5 \n", + "104 http://www.bbc.co.uk/news/uk-politics-42712180 5 \n", + "105 http://www.bbc.co.uk/news/uk-england-london-42... 5 \n", + "\n", + " tags tags_len main_tag sub_tag \n", + "59 [sport, football] 2 sport football \n", + "61 [misc, entertainment, arts] 3 misc entertainment \n", + "62 [misc, health] 2 misc health \n", + "63 [misc, health] 2 misc health \n", + "64 [uk, england, london] 3 uk england \n", + "65 [uk, england, london] 3 uk england \n", + "66 [world, europe] 2 world europe \n", + "67 [sport, football] 2 sport football \n", + "70 [uk, england, nottinghamshire] 3 uk england \n", + "71 [world, us, canada] 3 world us \n", + "72 [uk, england, berkshire] 3 uk england \n", + "73 [misc, technology] 2 misc technology \n", + "74 [uk, england] 2 uk england \n", + "75 [misc, health] 2 misc health \n", + "76 [misc, entertainment, arts] 3 misc entertainment \n", + "77 [misc, entertainment, arts] 3 misc entertainment \n", + "79 [world, europe] 2 world europe \n", + "81 [misc, business] 2 misc business \n", + "82 [uk, england, hereford, worcester] 4 uk england \n", + "84 [world, us, canada] 3 world us \n", + "85 [uk, england] 2 uk england \n", + "87 [misc, technology] 2 misc technology \n", + "88 [newsbeat, articles] 2 newsbeat articles \n", + "89 [misc, science, environment] 3 misc science \n", + "90 [uk, england, nottinghamshire] 3 uk england \n", + "91 [uk, england, bristol] 3 uk england \n", + "92 [misc, science, environment] 3 misc science \n", + "93 [misc, education] 2 misc education \n", + "94 [misc, business] 2 misc business \n", + "95 [misc, technology] 2 misc technology \n", + "96 [world, us, canada] 3 world us \n", + "97 [world, asia, pacific] 3 world asia \n", + "98 [world, us, canada] 3 world us \n", + "99 [uk, england, london] 3 uk england \n", + "100 [misc, entertainment, arts] 3 misc entertainment \n", + "101 [uk, england, merseyside] 3 uk england \n", + "102 [misc, technology] 2 misc technology \n", + "103 [uk, england] 2 uk england \n", + "104 [uk, politics] 2 uk politics \n", + "105 [uk, england, london] 3 uk england " + ] + }, + "execution_count": 153, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[50:90]" + ] + }, + { + "cell_type": "code", + "execution_count": 154, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "main_tag\n", + "bbcthree 10\n", + "bitesize 1\n", + "iplayer 12\n", + "misc 25448\n", + "newsbeat 11\n", + "sounds 25\n", + "sport 6299\n", + "uk 35916\n", + "weather 25\n", + "world 18658\n", + "dtype: int64" + ] + }, + "execution_count": 154, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# group by main tag and count the number of rows\n", + "df.groupby(\"main_tag\").size()" + ] + }, + { + "cell_type": "code", + "execution_count": 155, + "metadata": {}, + "outputs": [], + "source": [ + "# remove columns with main tags smaller than 1000\n", + "df = df.groupby(\"main_tag\").filter(lambda x: len(x) > 1000)" + ] + }, + { + "cell_type": "code", + "execution_count": 163, + "metadata": {}, + "outputs": [], + "source": [ + "# remove columns with sub tags smaller than 150\n", + "df = df.groupby([\"main_tag\", \"sub_tag\"]).filter(lambda x: len(x) > 150)" + ] + }, + { + "cell_type": "code", + "execution_count": 166, + "metadata": {}, + "outputs": [], + "source": [ + "# remove columns with sub tags \"in\", \"live\", \"newsbeat\"\n", + "df = df[~df[\"sub_tag\"].isin([\"in\", \"live\", \"newsbeat\"])]" + ] + }, + { + "cell_type": "code", + "execution_count": 156, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "main_tag\n", + "misc 25448\n", + "sport 6299\n", + "uk 35916\n", + "world 18658\n", + "dtype: int64" + ] + }, + "execution_count": 156, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.groupby(\"main_tag\").size()" + ] + }, + { + "cell_type": "code", + "execution_count": 157, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "main_tag\n", + "misc [newsbeat, education, science, entertainment, ...\n", + "sport [football, tennis, cricket, american-football,...\n", + "uk [england, politics, wales, scotland, northern]\n", + "world [asia, europe, latin, middle, us, africa, aust...\n", + "Name: sub_tag, dtype: object" + ] + }, + "execution_count": 157, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# group by main tag and print unique sub tags per main tag\n", + "df.groupby(\"main_tag\")[\"sub_tag\"].unique()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "main_tag sub_tag \n", + "misc business 8026\n", + " education 1175\n", + " election 614\n", + " entertainment 4776\n", + " health 2897\n", + " science 1817\n", + " technology 1963\n", + "sport athletics 182\n", + " boxing 158\n", + " cricket 500\n", + " football 3393\n", + " formula1 179\n", + " rugby 347\n", + " tennis 597\n", + "uk england 14475\n", + " northern-ireland 3433\n", + " politics 7590\n", + " scotland 5265\n", + " wales 5153\n", + "world africa 918\n", + " asia 2776\n", + " australia 700\n", + " europe 6332\n", + " latin-america 728\n", + " middle-east 1672\n", + " us 5516\n", + "dtype: int64" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# group by main tag and sub tag and count the number of rows\n", + "df.groupby([\"main_tag\", \"sub_tag\"]).size()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# rename subtag rugby-union to rugby\n", + "df[\"sub_tag\"] = df[\"sub_tag\"].replace(\"rugby-union\", \"rugby\")\n", + "\n", + "# rename subtag middle to middle-east\n", + "df[\"sub_tag\"] = df[\"sub_tag\"].replace(\"middle\", \"middle-east\")\n", + "\n", + "# rename subtag northern to northernireland\n", + "df[\"sub_tag\"] = df[\"sub_tag\"].replace(\"northern\", \"northern-ireland\")\n", + "\n", + "# rename subtag middle to middle-east\n", + "df[\"sub_tag\"] = df[\"sub_tag\"].replace(\"middle\", \"middle-east\")\n", + "\n", + "# rename subtag latin to latin-america\n", + "df[\"sub_tag\"] = df[\"sub_tag\"].replace(\"latin\", \"latin-america\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "# combine main tag and sub tag\n", + "df[\"tag\"] = df[\"main_tag\"] + \"/\" + df[\"sub_tag\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "# store the cleaned data in a new parquet file\n", + "df.to_parquet(\"bbc_cleaned.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
titlepublished_datedescriptionsectioncontentlinkcounttagstags_lenmain_tagsub_tagtag
0Kabul attack: Guests use sheets to escape hote...2018-01-21Guests used sheets to climb down from balconie...NoneGuests used sheets to climb down from balconie...http://www.bbc.co.uk/news/world-asia-427649715[world, asia]2worldasiaworld/asia
1Rashan Charles death: Met Police officer not f...2018-01-21Rashan Charles died as he tried to swallow a p...LondonRashan Charles died after being apprehended by...http://www.bbc.co.uk/news/uk-england-london-42...5[uk, england, london]3ukenglanduk/england
2Marco Silva: Watford blame Everton as they sac...2018-01-21Watford sack manager Marco Silva, blaming Ever...NoneWatford have sacked manager Marco Silva, blami...http://www.bbc.co.uk/sport/football/427658816[sport, football]2sportfootballsport/football
3North Korea Moranbong girl band leader heads O...2018-01-21The team is on a landmark visit to inspect cul...AsiaHyon Song-wol, pictured centre, was the star a...http://www.bbc.co.uk/news/world-asia-427651055[world, asia]2worldasiaworld/asia
4Australian Open: Kyle Edmund reaches first Gra...2018-01-21Britain's Kyle Edmund powers into a first Gran...NoneLast updated on .From the section Tennis\\n\\nCo...http://www.bbc.co.uk/sport/tennis/427643046[sport, tennis]2sporttennissport/tennis
\n", + "
" + ], + "text/plain": [ + " title published_date \\\n", + "0 Kabul attack: Guests use sheets to escape hote... 2018-01-21 \n", + "1 Rashan Charles death: Met Police officer not f... 2018-01-21 \n", + "2 Marco Silva: Watford blame Everton as they sac... 2018-01-21 \n", + "3 North Korea Moranbong girl band leader heads O... 2018-01-21 \n", + "4 Australian Open: Kyle Edmund reaches first Gra... 2018-01-21 \n", + "\n", + " description section \\\n", + "0 Guests used sheets to climb down from balconie... None \n", + "1 Rashan Charles died as he tried to swallow a p... London \n", + "2 Watford sack manager Marco Silva, blaming Ever... None \n", + "3 The team is on a landmark visit to inspect cul... Asia \n", + "4 Britain's Kyle Edmund powers into a first Gran... None \n", + "\n", + " content \\\n", + "0 Guests used sheets to climb down from balconie... \n", + "1 Rashan Charles died after being apprehended by... \n", + "2 Watford have sacked manager Marco Silva, blami... \n", + "3 Hyon Song-wol, pictured centre, was the star a... \n", + "4 Last updated on .From the section Tennis\\n\\nCo... \n", + "\n", + " link count \\\n", + "0 http://www.bbc.co.uk/news/world-asia-42764971 5 \n", + "1 http://www.bbc.co.uk/news/uk-england-london-42... 5 \n", + "2 http://www.bbc.co.uk/sport/football/42765881 6 \n", + "3 http://www.bbc.co.uk/news/world-asia-42765105 5 \n", + "4 http://www.bbc.co.uk/sport/tennis/42764304 6 \n", + "\n", + " tags tags_len main_tag sub_tag tag \n", + "0 [world, asia] 2 world asia world/asia \n", + "1 [uk, england, london] 3 uk england uk/england \n", + "2 [sport, football] 2 sport football sport/football \n", + "3 [world, asia] 2 world asia world/asia \n", + "4 [sport, tennis] 2 sport tennis sport/tennis " + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/benchmarks/datasets/coarsediscourse/.gitignore b/benchmarks/datasets/coarsediscourse/.gitignore new file mode 100644 index 000000000..94f335824 --- /dev/null +++ b/benchmarks/datasets/coarsediscourse/.gitignore @@ -0,0 +1,3 @@ +*.parquet +*.txt +*.json diff --git a/benchmarks/datasets/coarsediscourse/README.md b/benchmarks/datasets/coarsediscourse/README.md new file mode 100644 index 000000000..b26122f69 --- /dev/null +++ b/benchmarks/datasets/coarsediscourse/README.md @@ -0,0 +1,34 @@ +# Coarse Discourse Sequence Corpus + +Paper: https://ojs.aaai.org/index.php/ICWSM/article/view/14886 +Download: https://convokit.cornell.edu/documentation/coarseDiscourse.html + +## Setup +Run coarsediscourse.iypnb to download and preprocess the dataset. + +## What is Coarse Discourse Sequence Corpus? +Coarse Discourse, the Reddit dataset that contains ~9K threads, with comments annotated with 9 main discourse act labels and an “other” label. +Speakers in this Corpus are Reddit users, with their name being their Reddit username. +Each utterance represents either a top-level Reddit post or a comment on a post. + +## Labels +Comments are annotated with 9 main discourse act labels and an “other” label. +Discourse Act Definitions are taken from the paper. +``` +{"QUESTION": "A comment with a question or a request seeking some form of feedback, help, or other kinds of responses. While the comment may contain a question mark, it is not required. For instance, it might be posed in the form of a statement but still soliciting a response. Also, not everything that has a question mark is automatically a QUESTION. For instance, rhetorical questions are not seeking a response. Relation: This comment might be the first in a thread and have no relation to another comment. Or, it could be a clarifying or followup QUESTION linking to any prior comment.", +"ANSWER": "A comment that is responding to a QUESTION by answering the question or fulfilling the request. There can be more than one ANSWER responding to a QUESTION. Relation: An ANSWER is always linked to a QUESTION.", +"ANNOUNCEMENT": "A comment that is presenting some new information to the community, such as a piece of news, a link to something, a story, an opinion, a review, or insight. Relation: This comment has no relation to a prior comment and is always the initial post in a thread.", +"AGREEMENT": "A comment that is expressing agreement with some information presented in a prior comment. It can be agreeing with a point made, providing supporting evidence, providing a positive example or experience, or confirming or acknowledging a point made. Relation: This comment is always linked to a prior comment to which it is agreeing.", +"APPRECIATION": "A comment that is expressing thanks, appreciation, excitement, or praise in response to another comment. In contrast to AGREEMENT, it is not evaluating the merits of the points brought up. Comments of this category are more interpersonal as opposed to informational. Relation: This comment is always linked to a prior comment for which it is expressing appreciation.", +"DISAGREEMENT": "A comment that is correcting, criticizing, contradicting, or objecting to a point made in a prior comment. It can also be providing evidence to support its disagreement, such as an example or contrary anecdote.Relation: This comment is always linked to a prior comment to which it is disagreeing.", +"NEGATIVEREACTION": "A comment that is expressing a negative reaction to a previous comment, such as attacking or mocking the commenter, or expressing emotions like disgust, derision, or anger, to the contents of the prior comment. This comment is not discussing the merits of the points made in a prior comment or trying to correct them. Relation: This comment is always linked to a prior comment to which it is negatively reacting.", +"ELABORATION": "A comment that is adding additional information on to another comment. Oftentimes, one can imagine it simply appended to the end of the comment it elaborates on. One can elaborate on many kinds of comments, for instance, a questionasker elaborating on their question to provide more context, or someone elaborating on an answer to add more information. Relation: This comment is always linked to a prior comment upon which it is elaborating.", +"HUMOR": "This comment is primarily a joke, a piece of sarcasm, or a pun intended to get a laugh or be silly but not trying to add information. If a comment is sarcastic but using sarcasm to make a point or provide feedback, then it may belong in a different category. Relation: At times, this comment links to another comment but other times it may not be responding to anything." +"OTHER: "A comment that does not fit any of the previous definitions." +} +``` +{'negativereaction', 'disagreement', 'question', 'answer', 'announcement', 'other', 'appreciation', 'elaboration', 'agreement', 'humor'} + + +## Annotation +Three annotators were assigned to each thread and were instructed to annotate each comment in the thread with its discourse act (main_type) as well as the relation of each comment to a prior comment (link_to_post), if it existed. diff --git a/benchmarks/datasets/coarsediscourse/coarsediscourse.ipynb b/benchmarks/datasets/coarsediscourse/coarsediscourse.ipynb new file mode 100644 index 000000000..130f6a9c3 --- /dev/null +++ b/benchmarks/datasets/coarsediscourse/coarsediscourse.ipynb @@ -0,0 +1,145 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from convokit import Corpus, download\n", + "from typing import List\n", + "from pathlib import Path\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "datasets_path = Path(\"./datasets/coarsediscourse\")\n", + "test_path = datasets_path / \"coursediscourse_test.parquet\"\n", + "train_path = datasets_path / \"coursediscourse_train.parquet\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "corpus = Corpus(filename=download(\"reddit-coarse-discourse-corpus\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "sentences_list: List[List[str]] = []\n", + "labels_list: List[List[List[str]]] = [] \n", + "\n", + "for conversation in corpus.iter_conversations():\n", + "\n", + " sentences = []\n", + " labels = []\n", + "\n", + " speaker_map = {speaker_id: f\"Speaker {(idx + 1)}\" for idx, speaker_id in enumerate(conversation.get_speaker_ids())}\n", + " for utterance in conversation.iter_utterances():\n", + " text = utterance.text\n", + " text = \" \".join([text_segment for text_segment in text.split(\"\\n\") if len(text_segment.split()) > 1])\n", + " text = \" \".join(text.split(\"\\t\"))\n", + " text = \" \".join(text.split())\n", + "\n", + " sentences.append(text)\n", + " label = utterance.meta.get('majority_type', 'other')\n", + " if label is None:\n", + " label = 'other'\n", + " if label == \"negativereaction\":\n", + " label = \"negative reaction\"\n", + " labels.append(label)\n", + "\n", + " assert len(sentences) == len(labels), \"Number of labels and sentences do not match\"\n", + " sentences_list.append(sentences)\n", + " labels_list.append(labels)\n", + "\n", + "assert len(sentences_list) == len(labels_list), \"Number of labels and sentences do not match\"\n", + "\n", + "# create dataframe\n", + "df = pd.DataFrame({\"sentences\": sentences_list, \"labels\": labels_list})\n", + "\n", + "# unique labels\n", + "unique_labels = set()\n", + "for labels in labels_list:\n", + " unique_labels.update(labels)\n", + "print(unique_labels)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# split the df into training and test set\n", + "from sklearn.model_selection import train_test_split\n", + "\n", + "train_df, test_df = train_test_split(df, test_size=0.4, random_state=42)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# save dataframe\n", + "train_df.to_parquet(train_path)\n", + "test_df.to_parquet(test_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# read dataframe\n", + "train_df = pd.read_parquet(train_path)\n", + "test_df = pd.read_parquet(test_path)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sent-class", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.15" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/benchmarks/datasets/csabstruct/.gitignore b/benchmarks/datasets/csabstruct/.gitignore new file mode 100644 index 000000000..94f335824 --- /dev/null +++ b/benchmarks/datasets/csabstruct/.gitignore @@ -0,0 +1,3 @@ +*.parquet +*.txt +*.json diff --git a/benchmarks/datasets/csabstruct/README.md b/benchmarks/datasets/csabstruct/README.md new file mode 100644 index 000000000..8e7b03adb --- /dev/null +++ b/benchmarks/datasets/csabstruct/README.md @@ -0,0 +1,104 @@ +# CSAbstruct + +Paper: https://aclanthology.org/D19-1383/ +Download: https://huggingface.co/datasets/allenai/csabstruct + +## Setup +Run csabstruct.iypnb to download and preprocess the dataset. + +## What is CSAbstruct + +CSAbstruct is a dataset of annotated computer science abstracts with sentence labels according to their rhetorical roles. +CSAbstruct is collected from the Semantic Scholar corpus. +Each sentence is annotated by 5 workers, with one of 5 categories {BACKGROUND, OBJECTIVE, METHOD, RESULT, OTHER} + +## Statistics + +Label % in Dataset +BACKGROUND 33% +METHOD 32% +RESULT 21% +OBJECTIVE 12% +OTHER 03% + +Statistic Avg ± std +Doc length in sentences 6.7 ± 1.99 +Sentence length in words 21.8 ± 10.0 + +## Labels + +The label definitions are written by us. +Their paper does not provide any further definitions of the labels. +We use the same definitions in pubmed200k and csabstruct. + +``` +label_dict = { + "background": "Provides context or previous knowledge relevant to the research topic. Think of it as setting the stage for the study.", + "method": "Describes the procedures and techniques used in the research. This includes the study design, data collection, and analysis methods.", + "objective": "States the main goal or purpose of the research. What question is this work trying to answer?", + "result": "Presents the findings or outcomes of the research. This often includes statistical data, tables, and figures.", + "other": "Any sentence that doesn't fit into the above categories. This could be discussion, analysis, limitations, or concluding remarks.", +} +``` + +{'background', 'objective', 'method', 'result', 'other'} + +## Prompts + +### System Prompt + +# Your Role + +You are a professional annotator specialized in annotating sequences of sentences of a document with the help of provided annotation guidelines. +You are always strictly adhering to annotation guidelines making you a valuable partner in all research endevours. +Further, you always stick to the desired output format. + +You will be given the following information for every annotation task: + +- Document: The document to annotate split into a numbered sequence of sentences. +- Additional Instructions: Optionally, the user may provide specific details about the task. + +# Project Details + +You are a member of the project '{}'. +This project is about {}. +The project's success depends on your contributions to the annotation process. We count on you! + +# Annotation Guidelines + +These annotation guidelines explain all categories in detail. +Make sure to use these guidelines during the annotation process. + +{} + +# Your Strategy + +For all annotation tasks, it is crucial to go through the provided document sentence-by-sentence: + +1. Read the sentence carefully. +2. Think and reason which category fits the sentence best. +3. Classify the sentence based on the sentence itself and your reasoning. + +# Output Format + +In accordance with your strategy, you will answer in the following format for every provided sentence: + - - + +e.g. +1 - Fits the definition of Category A - A +2 - Meets the description of Category C - C +3 - Is similar to the definition of Category B - B +... + +It is required that you classify every provided sentence. +Therefore, the number of output sentences has to match the input sentences. + +### User Prompt + +Please annotate each sentence of the following document with the best fitting category of the Annotation Guidelines. + +Document: +{} + +Additional Instructions: +Remember to annotate every provided sentence. You are NOT ALLOWED to use any other category than those provided in the Annotation Guidelines! diff --git a/benchmarks/datasets/csabstruct/csabstruct.ipynb b/benchmarks/datasets/csabstruct/csabstruct.ipynb new file mode 100644 index 000000000..5feb47d4f --- /dev/null +++ b/benchmarks/datasets/csabstruct/csabstruct.ipynb @@ -0,0 +1,110 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datasets import load_dataset\n", + "from typing import List\n", + "\n", + "ds = load_dataset(\"allenai/csabstruct\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for split in ['train', 'validation', 'test']:\n", + " ds_split = ds[split]\n", + "\n", + " documents: List[List[str]] = ds_split[\"sentences\"] # type: ignore\n", + " labels = ds_split[\"labels\"] # type: ignore\n", + " id2label = {\n", + " 0: \"background\",\n", + " 1: \"method\",\n", + " 2: \"objective\",\n", + " 3: \"other\",\n", + " 4: \"result\"\n", + " }\n", + " labels = [[id2label[label] for label in lls] for lls in labels]\n", + "\n", + " # save as parquet\n", + " import pandas as pd\n", + "\n", + " df = pd.DataFrame({\n", + " \"sentences\": documents,\n", + " \"labels\": labels\n", + " })\n", + "\n", + " df.to_parquet(f\"datasets/csabstruct/{split}.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "label_dict = {\n", + " \"background\": \"Provides context or previous knowledge relevant to the research topic. Think of it as setting the stage for the study.\",\n", + " \"method\": \"Describes the procedures and techniques used in the research. This includes the study design, data collection, and analysis methods.\",\n", + " \"objective\": \"States the main goal or purpose of the research. What question is this work trying to answer?\",\n", + " \"other\": \"Any sentence that doesn't fit into the above categories. This could be discussion, analysis, limitations, or concluding remarks.\",\n", + " \"result\": \"Presents the findings or outcomes of the research. This often includes statistical data, tables, and figures.\",\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.read_parquet(\"datasets/csabstruct/train.parquet\")\n", + "documents: List[List[str]] = [list(document_sentences) for document_sentences in df[\"sentences\"].tolist()]\n", + "labels: List[List[str]] = [list(labels) for labels in df[\"labels\"].tolist()]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/benchmarks/datasets/daily_dialog/.gitignore b/benchmarks/datasets/daily_dialog/.gitignore new file mode 100644 index 000000000..94f335824 --- /dev/null +++ b/benchmarks/datasets/daily_dialog/.gitignore @@ -0,0 +1,3 @@ +*.parquet +*.txt +*.json diff --git a/benchmarks/datasets/daily_dialog/README.md b/benchmarks/datasets/daily_dialog/README.md new file mode 100644 index 000000000..de0216690 --- /dev/null +++ b/benchmarks/datasets/daily_dialog/README.md @@ -0,0 +1,54 @@ +# Daily Dialog Dataset + +Paper: https://aclanthology.org/I17-1099/ +Blog: http://yanran.li/dailydialog +Download: https://github.com/declare-lab/RECCON/tree/main/data/original_annotation + +## Setup +Run dailydialog.iypnb to preprocess the dataset. + +## What is Daily Dialog Dataset + +``` +Daily Topics: It covers ten categories ranging from ordinary life to financial topics, which is different from domain-specific datasets. + +Bi-turn Dialog Flow: It conforms basic dialog act flows, such as Questions-Inform and Directives-Commissives bi-turn flows, making it different from question answering (QA) datasets and post-reply datasets. + +Certain Communication Pattern: It follows unique multi-turn dialog flow patterns reflecting human communication style, which are rarely seen in task-oriented datasets. + +RichEmotion: It contains rich emotions and is labeled manually to keep high-quality, which is distinguished from most existing dialog datasets. +``` + +## Statistics + + Count ofEU ofTotal + +Anger 1022 5.87 0.99 +Disgust 353 2.03 0.34 +Fear 74 1.00 0.17 +Happiness 12885 74.02 12.51 +Sadness 1150 6.61 1.12 +Surpise 1823 10.47 1.77 +Other 85572 - 83.10 + +## Labels + +The labels follow the BigSix Theory by Paul Ekman 1992 - An argument for basic emotions. +The label definitions are written by us as the authors do not provide further details or descriptions. +We use the same definitions in dailydialog and emotion_lines. + +``` +{'fear': 'A feeling of apprehension or dread in response to a perceived threat or danger. It can range from mild anxiety to intense terror.', + 'disgust': 'A feeling of revulsion or aversion, often triggered by something perceived as unpleasant, unsanitary, or morally offensive.', + 'neutral': 'A state of emotional balance or equilibrium, where no particular emotion is dominant.', + 'anger': 'A feeling of intense displeasure or hostility, often triggered by a perceived wrong or injustice. It can manifest as irritation, frustration, rage, or fury.', + 'surprise': 'A brief emotional state in response to an unexpected event. It can be positive, negative, or neutral, depending on the nature of the surprise.', + 'sadness': 'A feeling of sorrow, grief, or disappointment. It can range from mild melancholy to intense despair.', + 'joy': 'A feeling of happiness, contentment, or pleasure. It can manifest as excitement, amusement or love.'} +``` + +{'fear', 'disgust', 'neutral', 'anger', 'surprise', 'sadness', 'joy'} + +## Evaluation + +Chosen labels: ? diff --git a/benchmarks/datasets/daily_dialog/dailydialog.ipynb b/benchmarks/datasets/daily_dialog/dailydialog.ipynb new file mode 100644 index 000000000..cfc0189cc --- /dev/null +++ b/benchmarks/datasets/daily_dialog/dailydialog.ipynb @@ -0,0 +1,249 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "from typing import List\n", + "import pandas as pd\n", + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "datasets_path = Path(\"./datasets\")\n", + "valid_path = datasets_path / \"daily_dialog\" / \"dailydialog_valid.json\"\n", + "test_path = datasets_path / \"daily_dialog\" / \"dailydialog_test.json\"\n", + "train_path = datasets_path / \"daily_dialog\" / \"dailydialog_train.json\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "label_map = {'happy': 'joy',\n", + " 'sad': 'sadness',\n", + " 'happines': 'joy', \n", + " 'disgust': 'disgust', \n", + " 'anger': 'anger',\n", + " 'excited': 'joy',\n", + " 'fear': 'fear',\n", + " 'surprised': 'surprise',\n", + " 'angry': 'anger', \n", + " 'neutral': 'neutral',\n", + " 'surprise': 'surprise', \n", + " 'sadness': 'sadness',\n", + " 'happiness': 'joy'\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def create_dataset(path: Path):\n", + " # read json file\n", + " data = json.loads(path.read_bytes())\n", + "\n", + " # extract sentences and labels\n", + " sentences_list: List[List[str]] = []\n", + " labels_list: List[List[str]] = []\n", + " for dialog_id, conversations in data.items():\n", + " if len(conversations) > 1:\n", + " print(f\"Dialog {dialog_id} has more than one conversation\")\n", + "\n", + " sentences: List[str] = []\n", + " labels: List[str] = []\n", + " for utterance in conversations[0]:\n", + " sentences.append(f\"Speaker {utterance['speaker']}: {utterance['utterance']}\")\n", + " labels.append(label_map[utterance[\"emotion\"]])\n", + "\n", + " sentences_list.append(sentences)\n", + " labels_list.append(labels)\n", + "\n", + " # create dataframe\n", + " df = pd.DataFrame({\"sentences\": sentences_list, \"labels\": labels_list})\n", + "\n", + " # save dataframe\n", + " df.to_parquet(path.with_suffix(\".parquet\"))\n", + "\n", + " # unique labels\n", + " unique_labels = set()\n", + " for labels in labels_list:\n", + " unique_labels.update(labels)\n", + " return unique_labels" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "l1 = create_dataset(valid_path)\n", + "l2 = create_dataset(test_path)\n", + "l3 = create_dataset(train_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "all_labels = l1.union(l2).union(l3)\n", + "print(all_labels)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# read datasets\n", + "valid_df = pd.read_parquet(valid_path.with_suffix(\".parquet\"))\n", + "test_df = pd.read_parquet(test_path.with_suffix(\".parquet\"))\n", + "train_df = pd.read_parquet(train_path.with_suffix(\".parquet\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "train_df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "valid_df[\"labels\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "len(train_df)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# count the labels\n", + "label_counts = train_df[\"labels\"].explode().value_counts()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "label_counts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# i want to find all rows that have a certain label\n", + "train_df[train_df[\"labels\"].apply(lambda x: \"fear\" in x)]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "train_df[\"labels\"][18]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# count the labels of train_df[\"labels\"][18]\n", + "train_df[\"labels\"][18].count(\"fear\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "label_list = list(train_df[\"labels\"][18])\n", + "\n", + "for idx, row in train_df[train_df[\"labels\"].apply(lambda x: \"fear\" in x)].iterrows():\n", + " label_list = list(row['labels'])\n", + " label_counts = {label: label_list.count(label) for label in set(label_list)}\n", + " print(idx)\n", + " print(label_counts)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "list(train_df[train_df[\"labels\"].apply(lambda x: \"fear\" in x)].iterrows())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sent-class", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.15" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/benchmarks/datasets/disco/.gitignore b/benchmarks/datasets/disco/.gitignore new file mode 100644 index 000000000..4226920f8 --- /dev/null +++ b/benchmarks/datasets/disco/.gitignore @@ -0,0 +1,4 @@ +* +!.gitignore +!README.md +!disco_dataset_creation.ipynb diff --git a/benchmarks/datasets/disco/README.md b/benchmarks/datasets/disco/README.md new file mode 100644 index 000000000..118867b1a --- /dev/null +++ b/benchmarks/datasets/disco/README.md @@ -0,0 +1,71 @@ +# DISCO (Disfluency Correction Dataset) + +## What Is This Dataset About? + +This dataset contains pairs of disfluent and fluent sentences for text rewrite/correction tasks. + +The local folder includes multilingual spreadsheet sources (`German.xlsx`, `English.xlsx`, plus additional language sheets). + +## Where Can It Be Found? + +- Local source files in this repository: + - `German.xlsx` + - `English.xlsx` + - `French.xlsx` + - `Hindi.xlsx` + - `Domain Type Distribution Sheet.xlsx` + +## Links (Website / Download / Citation) + +- The upstream download/citation link is not explicitly recorded in `disco_dataset_creation.ipynb`. +- Add the original release page and citation here once confirmed. + +## Benchmark Task Usage + +- Task 4.1: Disfluency Correction + +## Dataset Size (Current Files) + +- `disco_de.parquet`: 3096 samples +- `disco_en.parquet`: 3979 samples + +## Label Space + +`Disfluency Type` values available in both files: + +- `C` +- `F` +- `FL` +- `FS` +- `R` + +## How We Preprocess It + +Preprocessing is implemented in `disco_dataset_creation.ipynb`. + +Main steps: + +1. Load language-specific spreadsheets. +2. Keep core sentence-pair columns needed for correction. +3. Drop non-essential metadata/helper columns. +4. Export language-specific Parquet files for benchmark usage. + +## Final Dataset Structure + +### File: `disco_de.parquet` + +- `Sentence Number` +- `Disfluent Sentence` +- `Fluent Sentence` +- `Disfluency Type` + +### File: `disco_en.parquet` + +- `Sentence Number` +- `Disfluent Sentence` +- `Fluent Sentence` +- `Disfluency Type` + +## Notes + +- The notebook includes an output path named `disco_ger.parquet`; the current dataset folder contains `disco_de.parquet`. diff --git a/benchmarks/datasets/disco/disco_dataset_creation.ipynb b/benchmarks/datasets/disco/disco_dataset_creation.ipynb new file mode 100644 index 000000000..39005f6cc --- /dev/null +++ b/benchmarks/datasets/disco/disco_dataset_creation.ipynb @@ -0,0 +1,653 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.read_excel(\"datasets/disco/German.xlsx\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Sentence NumberDisfluent SentenceFluent SentenceDisfluency TypeEnglish Translation of Fluent SentenceDomain LabelUnnamed: 6DistirbutionUnnamed: 8
01Zeige mir Angebote zeigen.Mir Angebote zeigen.CShow me offers.OtherNaNOther89.0
12Pausiere mein mein Lauftraining.Pausiere mein Lauftraining.RPause my running training.Pause_exerciseNaNPause_exercise157.0
23Meine meine Mitfahrgelegenheit absagen.Meine Mitfahrgelegenheit absagen.RCancel my reide.Cancel_rideNaNCancel_ride109.0
34Pause pausiere das Laufen.Pausiere das Laufen.CPause the running.Pause_exerciseNaNStop_exercise170.0
45Beende beende meinen Workout.Beende meinen Workout.REnd my workout.Stop_exerciseNaNTake_photo140.0
..............................
30913092Übung Übung stoppen.Übung stoppen.RStop exercise.Stop_exerciseNaNNaNNaN
30923093Stoppe stoppe Workout.Stoppe Workout.RStop workout.Stop_exerciseNaNNaNNaN
30933094Post schreiben schreiben.Post schreiben.RWrite a post.Post_messageNaNNaNNaN
30943095Yoga beenden beenden.Yoga beenden.RFinish yoga.Stop_exerciseNaNNaNNaN
30953096Lauftraining Lauftraining starten.Lauftraining starten.RStart running practice.Start_exerciseNaNNaNNaN
\n", + "

3096 rows × 9 columns

\n", + "
" + ], + "text/plain": [ + " Sentence Number Disfluent Sentence \\\n", + "0 1 Zeige mir Angebote zeigen. \n", + "1 2 Pausiere mein mein Lauftraining. \n", + "2 3 Meine meine Mitfahrgelegenheit absagen. \n", + "3 4 Pause pausiere das Laufen. \n", + "4 5 Beende beende meinen Workout. \n", + "... ... ... \n", + "3091 3092 Übung Übung stoppen. \n", + "3092 3093 Stoppe stoppe Workout. \n", + "3093 3094 Post schreiben schreiben. \n", + "3094 3095 Yoga beenden beenden. \n", + "3095 3096 Lauftraining Lauftraining starten. \n", + "\n", + " Fluent Sentence Disfluency Type \\\n", + "0 Mir Angebote zeigen. C \n", + "1 Pausiere mein Lauftraining. R \n", + "2 Meine Mitfahrgelegenheit absagen. R \n", + "3 Pausiere das Laufen. C \n", + "4 Beende meinen Workout. R \n", + "... ... ... \n", + "3091 Übung stoppen. R \n", + "3092 Stoppe Workout. R \n", + "3093 Post schreiben. R \n", + "3094 Yoga beenden. R \n", + "3095 Lauftraining starten. R \n", + "\n", + " English Translation of Fluent Sentence Domain Label Unnamed: 6 \\\n", + "0 Show me offers. Other NaN \n", + "1 Pause my running training. Pause_exercise NaN \n", + "2 Cancel my reide. Cancel_ride NaN \n", + "3 Pause the running. Pause_exercise NaN \n", + "4 End my workout. Stop_exercise NaN \n", + "... ... ... ... \n", + "3091 Stop exercise. Stop_exercise NaN \n", + "3092 Stop workout. Stop_exercise NaN \n", + "3093 Write a post. Post_message NaN \n", + "3094 Finish yoga. Stop_exercise NaN \n", + "3095 Start running practice. Start_exercise NaN \n", + "\n", + " Distirbution Unnamed: 8 \n", + "0 Other 89.0 \n", + "1 Pause_exercise 157.0 \n", + "2 Cancel_ride 109.0 \n", + "3 Stop_exercise 170.0 \n", + "4 Take_photo 140.0 \n", + "... ... ... \n", + "3091 NaN NaN \n", + "3092 NaN NaN \n", + "3093 NaN NaN \n", + "3094 NaN NaN \n", + "3095 NaN NaN \n", + "\n", + "[3096 rows x 9 columns]" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# remove a few columns: English Translation of Fluent Sentence, Domain Label, Unnamed: 6, Distirbution, Unnamed: 8\n", + "df = df.drop(columns=[\"English Translation of Fluent Sentence\", \"Domain Label\", \"Unnamed: 6\", \"Distirbution\", \"Unnamed: 8\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Sentence NumberDisfluent SentenceFluent SentenceDisfluency Type
01Zeige mir Angebote zeigen.Mir Angebote zeigen.C
12Pausiere mein mein Lauftraining.Pausiere mein Lauftraining.R
23Meine meine Mitfahrgelegenheit absagen.Meine Mitfahrgelegenheit absagen.R
34Pause pausiere das Laufen.Pausiere das Laufen.C
45Beende beende meinen Workout.Beende meinen Workout.R
\n", + "
" + ], + "text/plain": [ + " Sentence Number Disfluent Sentence \\\n", + "0 1 Zeige mir Angebote zeigen. \n", + "1 2 Pausiere mein mein Lauftraining. \n", + "2 3 Meine meine Mitfahrgelegenheit absagen. \n", + "3 4 Pause pausiere das Laufen. \n", + "4 5 Beende beende meinen Workout. \n", + "\n", + " Fluent Sentence Disfluency Type \n", + "0 Mir Angebote zeigen. C \n", + "1 Pausiere mein Lauftraining. R \n", + "2 Meine Mitfahrgelegenheit absagen. R \n", + "3 Pausiere das Laufen. C \n", + "4 Beende meinen Workout. R " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "df.to_parquet(\"datasets/disco/disco_ger.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.read_excel(\"datasets/disco/English.xlsx\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Sentence NumberDisfluent SentenceFluent SentenceDisfluency TypeDomain LabelDistributionCount
00Send uhm send ETA via whatsappSend ETA via whatsappCSend_digital_objectSend_digital_object222.0
11Send te text to Heidi and include mom's ETA.Send text to Heidi and include mom's ETA.CSend_digital_objectGet_health_stats258.0
22Send a text to um 555-5555.Send a text to 555-5555.FSend_digital_objectGet_message_content191.0
33Send please send Nancy an SMS.please send Nancy an SMS.FSSend_digital_objectAdd_contact286.0
44I want to e-mail Zane this photo and cc um and...I want to e-mail Zane this photo and cc Zach.RSend_digital_objectCreate_note68.0
\n", + "
" + ], + "text/plain": [ + " Sentence Number Disfluent Sentence \\\n", + "0 0 Send uhm send ETA via whatsapp \n", + "1 1 Send te text to Heidi and include mom's ETA. \n", + "2 2 Send a text to um 555-5555. \n", + "3 3 Send please send Nancy an SMS. \n", + "4 4 I want to e-mail Zane this photo and cc um and... \n", + "\n", + " Fluent Sentence Disfluency Type \\\n", + "0 Send ETA via whatsapp C \n", + "1 Send text to Heidi and include mom's ETA. C \n", + "2 Send a text to 555-5555. F \n", + "3 please send Nancy an SMS. FS \n", + "4 I want to e-mail Zane this photo and cc Zach. R \n", + "\n", + " Domain Label Distribution Count \n", + "0 Send_digital_object Send_digital_object 222.0 \n", + "1 Send_digital_object Get_health_stats 258.0 \n", + "2 Send_digital_object Get_message_content 191.0 \n", + "3 Send_digital_object Add_contact 286.0 \n", + "4 Send_digital_object Create_note 68.0 " + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "# remove a few columns: Domain Label, Distribution, Count\n", + "df = df.drop(columns=[\"Domain Label\", \"Distribution\", \"Count\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Sentence NumberDisfluent SentenceFluent SentenceDisfluency Type
00Send uhm send ETA via whatsappSend ETA via whatsappC
11Send te text to Heidi and include mom's ETA.Send text to Heidi and include mom's ETA.C
22Send a text to um 555-5555.Send a text to 555-5555.F
33Send please send Nancy an SMS.please send Nancy an SMS.FS
44I want to e-mail Zane this photo and cc um and...I want to e-mail Zane this photo and cc Zach.R
\n", + "
" + ], + "text/plain": [ + " Sentence Number Disfluent Sentence \\\n", + "0 0 Send uhm send ETA via whatsapp \n", + "1 1 Send te text to Heidi and include mom's ETA. \n", + "2 2 Send a text to um 555-5555. \n", + "3 3 Send please send Nancy an SMS. \n", + "4 4 I want to e-mail Zane this photo and cc um and... \n", + "\n", + " Fluent Sentence Disfluency Type \n", + "0 Send ETA via whatsapp C \n", + "1 Send text to Heidi and include mom's ETA. C \n", + "2 Send a text to 555-5555. F \n", + "3 please send Nancy an SMS. FS \n", + "4 I want to e-mail Zane this photo and cc Zach. R " + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "df.to_parquet(\"datasets/disco/disco_en.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/benchmarks/datasets/disfl-qa/.gitignore b/benchmarks/datasets/disfl-qa/.gitignore new file mode 100644 index 000000000..212dc1b50 --- /dev/null +++ b/benchmarks/datasets/disfl-qa/.gitignore @@ -0,0 +1,4 @@ +* +!.gitignore +!README.md +!disflqa_dataset_creation.ipynb diff --git a/benchmarks/datasets/disfl-qa/README.md b/benchmarks/datasets/disfl-qa/README.md new file mode 100644 index 000000000..db9603625 --- /dev/null +++ b/benchmarks/datasets/disfl-qa/README.md @@ -0,0 +1,49 @@ +# Disfl-QA (Benchmark Dataset) + +## What Is This Dataset About? + +Disfl-QA contains disfluent question variants and their fluent/original versions. + +It can be used to evaluate robustness to spoken-style disfluencies and to evaluate correction/normalization quality for question inputs. + +## Where Can It Be Found? + +- Local files in this repository: + - `train.json` + - `dev.json` + - `test.json` + +## Links (Website / Download / Citation) + +- Dataset repository: + - https://github.com/google-research-datasets/Disfl-QA +- Paper: + - https://aclanthology.org/2021.findings-acl.293/ + +## Benchmark Task Usage + +- Primary: Task 4.1 Disfluency Correction (question normalization) +- Secondary/related: QA robustness experiments + +## Dataset Size (Current Files) + +- `disfl_qa_test.parquet` (main benchmark file): 3643 samples + +## How We Preprocess It + +Preprocessing is implemented in `disflqa_dataset_creation.ipynb`. + +Main steps: + +1. Load `test.json`. +2. Flatten dictionary-style entries into row-wise records. +3. Preserve original QA item ID as `id`. +4. Export test set to `disfl_qa_test.parquet`. + +## Final Dataset Structure + +### File: `disfl_qa_test.parquet` + +- `original`: fluent/original question text +- `disfluent`: disfluent question text +- `id`: dataset record identifier diff --git a/benchmarks/datasets/disfl-qa/disflqa_dataset_creation.ipynb b/benchmarks/datasets/disfl-qa/disflqa_dataset_creation.ipynb new file mode 100644 index 000000000..7740b8495 --- /dev/null +++ b/benchmarks/datasets/disfl-qa/disflqa_dataset_creation.ipynb @@ -0,0 +1,193 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from pathlib import Path\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "file_path = Path(\"datasets/disfl-qa/test.json\")\n", + "\n", + "# read json file\n", + "with open(file_path, \"r\") as file_path:\n", + " data = json.load(file_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "result = []\n", + "for key, value in data.items():\n", + " item = {}\n", + " item.update(value)\n", + " item[\"id\"] = key\n", + " result.append(item)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'original': 'In what country is Normandy located?',\n", + " 'disfluent': 'In what country is Norse found no wait Normandy not Norse?',\n", + " 'id': '56ddde6b9a695914005b9628'}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "result[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.DataFrame(result)\n", + "df.to_parquet(\"datasets/disfl-qa/disfl_qa_test.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
originaldisfluentid
0In what country is Normandy located?In what country is Norse found no wait Normand...56ddde6b9a695914005b9628
1When were the Normans in Normandy?From which countries no tell me when were the ...56ddde6b9a695914005b9629
2From which countries did the Norse originate?From which Norse leader I mean countries did t...56ddde6b9a695914005b962a
3Who was the Norse leader?When I mean Who was the Norse leader?56ddde6b9a695914005b962b
4What century did the Normans first gain their ...When no what century did the Normans first gai...56ddde6b9a695914005b962c
\n", + "
" + ], + "text/plain": [ + " original \\\n", + "0 In what country is Normandy located? \n", + "1 When were the Normans in Normandy? \n", + "2 From which countries did the Norse originate? \n", + "3 Who was the Norse leader? \n", + "4 What century did the Normans first gain their ... \n", + "\n", + " disfluent id \n", + "0 In what country is Norse found no wait Normand... 56ddde6b9a695914005b9628 \n", + "1 From which countries no tell me when were the ... 56ddde6b9a695914005b9629 \n", + "2 From which Norse leader I mean countries did t... 56ddde6b9a695914005b962a \n", + "3 When I mean Who was the Norse leader? 56ddde6b9a695914005b962b \n", + "4 When no what century did the Normans first gai... 56ddde6b9a695914005b962c " + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/benchmarks/datasets/emotion_lines/.gitignore b/benchmarks/datasets/emotion_lines/.gitignore new file mode 100644 index 000000000..94f335824 --- /dev/null +++ b/benchmarks/datasets/emotion_lines/.gitignore @@ -0,0 +1,3 @@ +*.parquet +*.txt +*.json diff --git a/benchmarks/datasets/emotion_lines/README.md b/benchmarks/datasets/emotion_lines/README.md new file mode 100644 index 000000000..182eb6eab --- /dev/null +++ b/benchmarks/datasets/emotion_lines/README.md @@ -0,0 +1,40 @@ +# Emotion Lines Dataset + +Paper: https://aclanthology.org/L18-1252/ +Download: https://doraemon.iis.sinica.edu.tw/emotionlines/download.html + +## Setup +Run emotion_lines.iypnb to preprocess the dataset. + +## What is EmotionLines? +``` +We introduce EmotionLines, the first dataset with emotions labeling on all utterances in each dialogue only based on their textual content. +Dialogues in EmotionLines are collected from Friends TV scripts and private Facebook messenger dialogues. +Then one of seven emotions, six Ekman’s basic emotions plus the neutral emotion, is labeled on each utterance by 5 Amazon MTurkers. +A total of 29,245 utterances from 2,000 dialogues are labeled in EmotionLines +``` + +## Labels +The labels follow the BigSix Theory by Paul Ekman 1992 - An argument for basic emotions. Cognition & emotion + +Labels: neutral, joy, sadness, fear, anger, surprise, disgust, (non-neutral) + +non-neutral: Each HIT was accomplished by 5 workers, and for each utterance in a HIT, the emotion with the highest number of votes was set as the gold label of the utterance. Those utterances with more than two different emotions voted were put into the non-neutral category. + +``` +{'fear': 'A feeling of apprehension or dread in response to a perceived threat or danger. It can range from mild anxiety to intense terror.', + 'disgust': 'A feeling of revulsion or aversion, often triggered by something perceived as unpleasant, unsanitary, or morally offensive.', + 'neutral': 'A state of emotional balance or equilibrium, where no particular emotion is dominant.', + 'anger': 'A feeling of intense displeasure or hostility, often triggered by a perceived wrong or injustice. It can manifest as irritation, frustration, rage, or fury.', + 'surprise': 'A brief emotional state in response to an unexpected event. It can be positive, negative, or neutral, depending on the nature of the surprise.', + 'sadness': 'A feeling of sorrow, grief, or disappointment. It can range from mild melancholy to intense despair.', + 'joy': 'A feeling of happiness, contentment, or pleasure. It can manifest as excitement, amusement, or love.', + 'non-neutral': 'Use this label if other or multiple of the above emotions are present' + } +``` + +{'surprise', 'sadness', 'fear', 'disgust', 'neutral', 'anger', 'non-neutral', 'joy'} + +## Evaluation +Chosen labels: joy, sadness, anger, and neutral. +non-neutral will be ignored diff --git a/benchmarks/datasets/emotion_lines/emotion_lines.ipynb b/benchmarks/datasets/emotion_lines/emotion_lines.ipynb new file mode 100644 index 000000000..cae178e46 --- /dev/null +++ b/benchmarks/datasets/emotion_lines/emotion_lines.ipynb @@ -0,0 +1,155 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "from typing import List\n", + "import pandas as pd\n", + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "datasets_path = Path(\"./datasets\")\n", + "dev_path = datasets_path / \"emotion_lines\" / \"friends_dev.json\"\n", + "test_path = datasets_path / \"emotion_lines\" / \"friends_test.json\"\n", + "train_path = datasets_path / \"emotion_lines\" / \"friends_train.json\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def create_dataset(path: Path):\n", + " # read json file\n", + " data = json.loads(path.read_bytes())\n", + "\n", + " # extract sentences and labels\n", + " sentences_list: List[List[str]] = []\n", + " labels_list: List[List[str]] = []\n", + " for dialog in data:\n", + " sentences: List[str] = []\n", + " labels: List[str] = []\n", + " for utterance in dialog:\n", + " sentences.append(f\"{utterance['speaker']}: {utterance['utterance']}\")\n", + " labels.append(utterance[\"emotion\"])\n", + "\n", + " sentences_list.append(sentences)\n", + " labels_list.append(labels)\n", + "\n", + " # create dataframe\n", + " df = pd.DataFrame({\"sentences\": sentences_list, \"labels\": labels_list})\n", + "\n", + " # save dataframe\n", + " df.to_parquet(path.with_suffix(\".parquet\"))\n", + "\n", + " # unique labels\n", + " unique_labels = set()\n", + " for labels in labels_list:\n", + " unique_labels.update(labels)\n", + " return unique_labels" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "l1 = create_dataset(dev_path)\n", + "l2 = create_dataset(test_path)\n", + "l3 = create_dataset(train_path)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "all_labels = l1.union(l2).union(l3)\n", + "print(all_labels)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# read datasets\n", + "dev_df = pd.read_parquet(dev_path.with_suffix(\".parquet\"))\n", + "test_df = pd.read_parquet(test_path.with_suffix(\".parquet\"))\n", + "train_df = pd.read_parquet(train_path.with_suffix(\".parquet\"))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "test_df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# i want to count the number of rows that contain the label \"non-neutral\"\n", + "\n", + "# count the number of rows that contain the label \"non-neutral\"\n", + "print(dev_df[\"labels\"].apply(lambda x: \"non-neutral\" in x).sum())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "len(test_df)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sent-class", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.15" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/benchmarks/datasets/fewnerd/.gitignore b/benchmarks/datasets/fewnerd/.gitignore new file mode 100644 index 000000000..857b83d60 --- /dev/null +++ b/benchmarks/datasets/fewnerd/.gitignore @@ -0,0 +1,4 @@ +* +!.gitignore +!README.md +!fewnerd_dataset_creation.py diff --git a/benchmarks/datasets/fewnerd/README.md b/benchmarks/datasets/fewnerd/README.md new file mode 100644 index 000000000..23250f9af --- /dev/null +++ b/benchmarks/datasets/fewnerd/README.md @@ -0,0 +1,43 @@ +# Few-NERD (Benchmark Dataset) + +## What Is This Dataset About? + +Few-NERD is an English named entity recognition dataset with coarse and fine-grained entity labels. + +In this benchmark, it is used for token/span labeling. + +## Where Can It Be Found? + +- Hugging Face dataset: + - https://huggingface.co/datasets/DFKI-SLT/few-nerd + +## Links (Website / Download / Citation) + +- Dataset repository: + - https://github.com/thunlp/Few-NERD +- Hugging Face dataset card: + - https://huggingface.co/datasets/DFKI-SLT/few-nerd + +## Benchmark Task Usage + +- Task 3: Span Classification + +## How We Preprocess It + +Preprocessing is implemented in `fewnerd_dataset_creation.py`. + +Main steps: + +1. Load the `supervised` configuration from Hugging Face. +2. Select the `test` split. +3. Save the split to parquet without additional sampling or filtering. + +## Final Dataset Structure + +### File: `fewnerd_test.parquet` + +The parquet file preserves original dataset columns, including at least: + +- `tokens`: token sequence +- `ner_tags`: coarse tag sequence (integer IDs) +- `fine_ner_tags`: fine-grained tag sequence (integer IDs) diff --git a/benchmarks/datasets/fewnerd/fewnerd_dataset_creation.py b/benchmarks/datasets/fewnerd/fewnerd_dataset_creation.py new file mode 100644 index 000000000..d6314e272 --- /dev/null +++ b/benchmarks/datasets/fewnerd/fewnerd_dataset_creation.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from pathlib import Path + +from datasets import load_dataset + + +def create_fewnerd_dataset(output_path: Path) -> None: + dataset = load_dataset("DFKI-SLT/few-nerd", "supervised") + df = dataset["test"].to_pandas() + + output_path.parent.mkdir(parents=True, exist_ok=True) + df.to_parquet(output_path, index=False) + + print("Few-NERD dataset download completed.") + print(f"Rows: {len(df)} -> {output_path}") + + +def main() -> None: + project_root = Path(__file__).resolve().parents[2] + output_path = project_root / "datasets/fewnerd/fewnerd_test.parquet" + create_fewnerd_dataset(output_path=output_path) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/datasets/german-ler/.gitignore b/benchmarks/datasets/german-ler/.gitignore new file mode 100644 index 000000000..de458b0a7 --- /dev/null +++ b/benchmarks/datasets/german-ler/.gitignore @@ -0,0 +1,4 @@ +* +!.gitignore +!README.md +!german_ler_creation.ipynb diff --git a/benchmarks/datasets/german-ler/README.md b/benchmarks/datasets/german-ler/README.md new file mode 100644 index 000000000..feeb25ca4 --- /dev/null +++ b/benchmarks/datasets/german-ler/README.md @@ -0,0 +1,87 @@ +# German-LER (Benchmark Dataset) + +## What Is This Dataset About? + +German-LER is a German-language named entity recognition dataset. + +In this benchmark, it is used for token/span labeling (BIO tagging) with both fine-grained and coarse-grained labels. + +## Where Can It Be Found? + +- Hugging Face dataset: + - https://huggingface.co/datasets/elenanereiss/german-ler + +## Links (Website / Download / Citation) + +- Dataset card: + - https://huggingface.co/datasets/elenanereiss/german-ler + +## Benchmark Task Usage + +- Task 3: Span Classification + +## Dataset Size (Current Files) + +- `german_ler_test.parquet` (main benchmark file): 6673 samples + +## How We Preprocess It + +Preprocessing is implemented in `german_ler_creation.ipynb`. + +Main steps: + +1. Load dataset via `load_dataset("elenanereiss/german-ler")`. +2. Work from the test split for benchmark evaluation. +3. Preserve tokenized input sequence. +4. Generate/keep both label granularities: + - `fine_ner_tags` + - `ner_tags` (coarser mapping) +5. Convert labels to integer ID sequences suitable for modeling/evaluation. +6. Save result to `german_ler_test.parquet`. + +## Final Dataset Structure + +### File: `german_ler_test.parquet` + +- `id`: sample identifier +- `tokens`: token sequence +- `ner_tags`: coarse BIO tag sequence (integer IDs) +- `fine_ner_tags`: fine-grained BIO tag sequence (integer IDs) + +## Tag Definitions Used In The Final Dataset + +The final Parquet stores integer ID sequences. During preprocessing, IDs map to merged semantic labels as follows. + +### `ner_tags` (coarse) ID -> label + +- `0`: `O` +- `1`: `Person` +- `2`: `Ort` +- `3`: `Organisation` +- `4`: `Norm` +- `5`: `Gesetz` +- `6`: `Rechtsprechung` +- `7`: `Literatur` + +### `fine_ner_tags` (fine) ID -> label + +- `0`: `O` +- `1`: `Person` +- `2`: `Anwalt` +- `3`: `Richter` +- `4`: `Land` +- `5`: `Stadt` +- `6`: `Straße` +- `7`: `Landschaft` +- `8`: `Organisation` +- `9`: `Unternehmen` +- `10`: `Institution` +- `11`: `Gericht` +- `12`: `Marke` +- `13`: `Gesetz` +- `14`: `Verordnung` +- `15`: `EU Norm` +- `16`: `Vorschrift` +- `17`: `Vertrag` +- `18`: `Rechtsprechung` +- `19`: `Literatur` diff --git a/benchmarks/datasets/german-ler/german_ler_creation.ipynb b/benchmarks/datasets/german-ler/german_ler_creation.ipynb new file mode 100644 index 000000000..78e45b6a0 --- /dev/null +++ b/benchmarks/datasets/german-ler/german_ler_creation.ipynb @@ -0,0 +1,1235 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/tfischer/micromamba/envs/dwts/lib/python3.11/site-packages/datasets/load.py:1486: FutureWarning: The repository for elenanereiss/german-ler contains custom code which must be executed to correctly load the dataset. You can inspect the repository content at https://hf.co/datasets/elenanereiss/german-ler\n", + "You can avoid this message in future by passing the argument `trust_remote_code=True`.\n", + "Passing `trust_remote_code=True` will be mandatory to load this dataset from the next major release of `datasets`.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "from datasets import load_dataset\n", + "ds = load_dataset(\"elenanereiss/german-ler\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "df = ds[\"test\"].to_pandas()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idtokensner_tagsner_coarse_tags
00[Wegen, der, Teilnahme, des, Antragstellers, a...[38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 3...[14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1...
11[●, Mitwirkung, im, Sinne, der, Kostenverfügung][38, 38, 38, 38, 38, 38][14, 14, 14, 14, 14, 14]
22[Von, der, Ablehnung, eines, Straferlasses, fü...[38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 3...[14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1...
33[Zwar, führt, bei, der, in, §, 33, Abs., 2, TV...[38, 38, 38, 38, 38, 18, 37, 37, 37, 37, 37, 3...[14, 14, 14, 14, 14, 5, 12, 12, 12, 12, 12, 12...
44[Der, Wortlaut, der, Zulagenregelung, verlange...[38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 3...[14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1...
\n", + "
" + ], + "text/plain": [ + " id tokens \\\n", + "0 0 [Wegen, der, Teilnahme, des, Antragstellers, a... \n", + "1 1 [●, Mitwirkung, im, Sinne, der, Kostenverfügung] \n", + "2 2 [Von, der, Ablehnung, eines, Straferlasses, fü... \n", + "3 3 [Zwar, führt, bei, der, in, §, 33, Abs., 2, TV... \n", + "4 4 [Der, Wortlaut, der, Zulagenregelung, verlange... \n", + "\n", + " ner_tags \\\n", + "0 [38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 3... \n", + "1 [38, 38, 38, 38, 38, 38] \n", + "2 [38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 3... \n", + "3 [38, 38, 38, 38, 38, 18, 37, 37, 37, 37, 37, 3... \n", + "4 [38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 3... \n", + "\n", + " ner_coarse_tags \n", + "0 [14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1... \n", + "1 [14, 14, 14, 14, 14, 14] \n", + "2 [14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1... \n", + "3 [14, 14, 14, 14, 14, 5, 12, 12, 12, 12, 12, 12... \n", + "4 [14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1... " + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "df[\"fine_ner_tags\"] = df[\"ner_tags\"]\n", + "df[\"ner_tags\"] = df[\"ner_coarse_tags\"]\n", + "del df[\"ner_coarse_tags\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idtokensner_tagsfine_ner_tags
00[Wegen, der, Teilnahme, des, Antragstellers, a...[14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1...[38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 3...
11[●, Mitwirkung, im, Sinne, der, Kostenverfügung][14, 14, 14, 14, 14, 14][38, 38, 38, 38, 38, 38]
22[Von, der, Ablehnung, eines, Straferlasses, fü...[14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1...[38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 3...
33[Zwar, führt, bei, der, in, §, 33, Abs., 2, TV...[14, 14, 14, 14, 14, 5, 12, 12, 12, 12, 12, 12...[38, 38, 38, 38, 38, 18, 37, 37, 37, 37, 37, 3...
44[Der, Wortlaut, der, Zulagenregelung, verlange...[14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1...[38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 3...
\n", + "
" + ], + "text/plain": [ + " id tokens \\\n", + "0 0 [Wegen, der, Teilnahme, des, Antragstellers, a... \n", + "1 1 [●, Mitwirkung, im, Sinne, der, Kostenverfügung] \n", + "2 2 [Von, der, Ablehnung, eines, Straferlasses, fü... \n", + "3 3 [Zwar, führt, bei, der, in, §, 33, Abs., 2, TV... \n", + "4 4 [Der, Wortlaut, der, Zulagenregelung, verlange... \n", + "\n", + " ner_tags \\\n", + "0 [14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1... \n", + "1 [14, 14, 14, 14, 14, 14] \n", + "2 [14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1... \n", + "3 [14, 14, 14, 14, 14, 5, 12, 12, 12, 12, 12, 12... \n", + "4 [14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1... \n", + "\n", + " fine_ner_tags \n", + "0 [38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 3... \n", + "1 [38, 38, 38, 38, 38, 38] \n", + "2 [38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 3... \n", + "3 [38, 38, 38, 38, 38, 18, 37, 37, 37, 37, 37, 3... \n", + "4 [38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 38, 3... " + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "fine_labels=[\n", + " 'B-AN', \n", + " 'B-EUN', \n", + " 'B-GRT', \n", + " 'B-GS', \n", + " 'B-INN', \n", + " 'B-LD', \n", + " 'B-LDS', \n", + " 'B-LIT', \n", + " 'B-MRK', \n", + " 'B-ORG', \n", + " 'B-PER', \n", + " 'B-RR', \n", + " 'B-RS', \n", + " 'B-ST', \n", + " 'B-STR', \n", + " 'B-UN', \n", + " 'B-VO', \n", + " 'B-VS', \n", + " 'B-VT', \n", + " 'I-AN', \n", + " 'I-EUN', \n", + " 'I-GRT', \n", + " 'I-GS', \n", + " 'I-INN', \n", + " 'I-LD', \n", + " 'I-LDS', \n", + " 'I-LIT', \n", + " 'I-MRK', \n", + " 'I-ORG', \n", + " 'I-PER', \n", + " 'I-RR', \n", + " 'I-RS', \n", + " 'I-ST', \n", + " 'I-STR', \n", + " 'I-UN', \n", + " 'I-VO', \n", + " 'I-VS', \n", + " 'I-VT', \n", + " 'O'\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "id2fine_label = {i: label for i, label in enumerate(fine_labels)}" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 'B-AN',\n", + " 1: 'B-EUN',\n", + " 2: 'B-GRT',\n", + " 3: 'B-GS',\n", + " 4: 'B-INN',\n", + " 5: 'B-LD',\n", + " 6: 'B-LDS',\n", + " 7: 'B-LIT',\n", + " 8: 'B-MRK',\n", + " 9: 'B-ORG',\n", + " 10: 'B-PER',\n", + " 11: 'B-RR',\n", + " 12: 'B-RS',\n", + " 13: 'B-ST',\n", + " 14: 'B-STR',\n", + " 15: 'B-UN',\n", + " 16: 'B-VO',\n", + " 17: 'B-VS',\n", + " 18: 'B-VT',\n", + " 19: 'I-AN',\n", + " 20: 'I-EUN',\n", + " 21: 'I-GRT',\n", + " 22: 'I-GS',\n", + " 23: 'I-INN',\n", + " 24: 'I-LD',\n", + " 25: 'I-LDS',\n", + " 26: 'I-LIT',\n", + " 27: 'I-MRK',\n", + " 28: 'I-ORG',\n", + " 29: 'I-PER',\n", + " 30: 'I-RR',\n", + " 31: 'I-RS',\n", + " 32: 'I-ST',\n", + " 33: 'I-STR',\n", + " 34: 'I-UN',\n", + " 35: 'I-VO',\n", + " 36: 'I-VS',\n", + " 37: 'I-VT',\n", + " 38: 'O'}" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "id2fine_label" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "df[\"fine_ner_tags\"] = df[\"fine_ner_tags\"].apply(lambda x: [id2fine_label[i] for i in x])" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idtokensner_tagsfine_ner_tags
00[Wegen, der, Teilnahme, des, Antragstellers, a...[14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1...[O, O, O, O, O, O, O, O, O, O, O, O, O, O, B-L...
11[●, Mitwirkung, im, Sinne, der, Kostenverfügung][14, 14, 14, 14, 14, 14][O, O, O, O, O, O]
22[Von, der, Ablehnung, eines, Straferlasses, fü...[14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1...[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...
33[Zwar, führt, bei, der, in, §, 33, Abs., 2, TV...[14, 14, 14, 14, 14, 5, 12, 12, 12, 12, 12, 12...[O, O, O, O, O, B-VT, I-VT, I-VT, I-VT, I-VT, ...
44[Der, Wortlaut, der, Zulagenregelung, verlange...[14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1...[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O]
\n", + "
" + ], + "text/plain": [ + " id tokens \\\n", + "0 0 [Wegen, der, Teilnahme, des, Antragstellers, a... \n", + "1 1 [●, Mitwirkung, im, Sinne, der, Kostenverfügung] \n", + "2 2 [Von, der, Ablehnung, eines, Straferlasses, fü... \n", + "3 3 [Zwar, führt, bei, der, in, §, 33, Abs., 2, TV... \n", + "4 4 [Der, Wortlaut, der, Zulagenregelung, verlange... \n", + "\n", + " ner_tags \\\n", + "0 [14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1... \n", + "1 [14, 14, 14, 14, 14, 14] \n", + "2 [14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1... \n", + "3 [14, 14, 14, 14, 14, 5, 12, 12, 12, 12, 12, 12... \n", + "4 [14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1... \n", + "\n", + " fine_ner_tags \n", + "0 [O, O, O, O, O, O, O, O, O, O, O, O, O, O, B-L... \n", + "1 [O, O, O, O, O, O] \n", + "2 [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ... \n", + "3 [O, O, O, O, O, B-VT, I-VT, I-VT, I-VT, I-VT, ... \n", + "4 [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O] " + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "fine_labels_merge = {\n", + " 'B-AN': 'Anwalt',\n", + " 'B-EUN': 'EU Norm',\n", + " 'B-GRT': 'Gericht',\n", + " 'B-GS': 'Gesetz',\n", + " 'B-INN': 'Institution',\n", + " 'B-LD': 'Land',\n", + " 'B-LDS': 'Landschaft',\n", + " 'B-LIT': 'Literatur',\n", + " 'B-MRK': 'Marke',\n", + " 'B-ORG': 'Organisation',\n", + " 'B-PER': 'Person',\n", + " 'B-RR': 'Richter',\n", + " 'B-RS': 'Gerichtsentscheidung',\n", + " 'B-ST': 'Stadt',\n", + " 'B-STR': 'Straße',\n", + " 'B-UN': 'Unternehmen',\n", + " 'B-VO': 'Verordnung',\n", + " 'B-VS': 'Vorschrift',\n", + " 'B-VT': 'Vertrag',\n", + " 'I-AN': 'Anwalt',\n", + " 'I-EUN': 'EU Norm',\n", + " 'I-GRT': 'Gericht',\n", + " 'I-GS': 'Gesetz',\n", + " 'I-INN': 'Institution',\n", + " 'I-LD': 'Land',\n", + " 'I-LDS': 'Landschaft',\n", + " 'I-LIT': 'Literatur',\n", + " 'I-MRK': 'Marke',\n", + " 'I-ORG': 'Organisation', \n", + " 'I-PER': 'Person',\n", + " 'I-RR': 'Richter',\n", + " 'I-RS': 'Gerichtsentscheidung',\n", + " 'I-ST': 'Stadt',\n", + " 'I-STR': 'Straße',\n", + " 'I-UN': 'Unternehmen',\n", + " 'I-VO': 'Verordnung',\n", + " 'I-VS': 'Vorschrift',\n", + " 'I-VT': 'Vertrag',\n", + " 'O': 'O'\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "fine_labels_merge2id = {\n", + " 'O': 0,\n", + " 'Person': 1,\n", + " 'Anwalt': 2,\n", + " 'Richter': 3,\n", + " 'Land': 4,\n", + " 'Stadt': 5,\n", + " 'Straße': 6,\n", + " 'Landschaft': 7,\n", + " 'Organisation': 8, \n", + " 'Unternehmen': 9,\n", + " 'Institution': 10,\n", + " 'Gericht': 11,\n", + " 'Marke': 12,\n", + " 'Gesetz': 13,\n", + " 'Verordnung': 14,\n", + " 'EU Norm': 15,\n", + " 'Vorschrift': 16,\n", + " 'Vertrag': 17,\n", + " 'Rechtsprechung': 18,\n", + " 'Literatur': 19,\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 'O',\n", + " 1: 'Person',\n", + " 2: 'Anwalt',\n", + " 3: 'Richter',\n", + " 4: 'Land',\n", + " 5: 'Stadt',\n", + " 6: 'Straße',\n", + " 7: 'Landschaft',\n", + " 8: 'Organisation',\n", + " 9: 'Unternehmen',\n", + " 10: 'Institution',\n", + " 11: 'Gericht',\n", + " 12: 'Marke',\n", + " 13: 'Gesetz',\n", + " 14: 'Verordnung',\n", + " 15: 'EU Norm',\n", + " 16: 'Vorschrift',\n", + " 17: 'Vertrag',\n", + " 18: 'Gerichtsentscheidung',\n", + " 19: 'Literatur'}" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "fine_id2label = {i: label for i, label in enumerate(fine_labels_merge2id)}\n", + "fine_id2label" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "df[\"fine_ner_tags\"] = df[\"fine_ner_tags\"].apply(lambda x: [fine_labels_merge[i] for i in x])" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "df[\"fine_ner_tags\"] = df[\"fine_ner_tags\"].apply(lambda x: [fine_labels_merge2id[i] for i in x])" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idtokensner_tagsfine_ner_tags
00[Wegen, der, Teilnahme, des, Antragstellers, a...[14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, ...
11[●, Mitwirkung, im, Sinne, der, Kostenverfügung][14, 14, 14, 14, 14, 14][0, 0, 0, 0, 0, 0]
22[Von, der, Ablehnung, eines, Straferlasses, fü...[14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...
33[Zwar, führt, bei, der, in, §, 33, Abs., 2, TV...[14, 14, 14, 14, 14, 5, 12, 12, 12, 12, 12, 12...[0, 0, 0, 0, 0, 17, 17, 17, 17, 17, 17, 17, 0,...
44[Der, Wortlaut, der, Zulagenregelung, verlange...[14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
\n", + "
" + ], + "text/plain": [ + " id tokens \\\n", + "0 0 [Wegen, der, Teilnahme, des, Antragstellers, a... \n", + "1 1 [●, Mitwirkung, im, Sinne, der, Kostenverfügung] \n", + "2 2 [Von, der, Ablehnung, eines, Straferlasses, fü... \n", + "3 3 [Zwar, führt, bei, der, in, §, 33, Abs., 2, TV... \n", + "4 4 [Der, Wortlaut, der, Zulagenregelung, verlange... \n", + "\n", + " ner_tags \\\n", + "0 [14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1... \n", + "1 [14, 14, 14, 14, 14, 14] \n", + "2 [14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1... \n", + "3 [14, 14, 14, 14, 14, 5, 12, 12, 12, 12, 12, 12... \n", + "4 [14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 1... \n", + "\n", + " fine_ner_tags \n", + "0 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, ... \n", + "1 [0, 0, 0, 0, 0, 0] \n", + "2 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... \n", + "3 [0, 0, 0, 0, 0, 17, 17, 17, 17, 17, 17, 17, 0,... \n", + "4 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] " + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [], + "source": [ + "coarse_labels = [\n", + " 'B-LIT', \n", + " 'B-LOC', \n", + " 'B-NRM', \n", + " 'B-ORG', \n", + " 'B-PER', \n", + " 'B-REG', \n", + " 'B-RS', \n", + " 'I-LIT', \n", + " 'I-LOC', \n", + " 'I-NRM', \n", + " 'I-ORG', \n", + " 'I-PER', \n", + " 'I-REG', \n", + " 'I-RS', \n", + " 'O'\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [], + "source": [ + "id2coarse_label = {i: label for i, label in enumerate(coarse_labels)}" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "df[\"ner_tags\"] = df[\"ner_tags\"].apply(lambda x: [id2coarse_label[i] for i in x])" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idtokensner_tagsfine_ner_tags
00[Wegen, der, Teilnahme, des, Antragstellers, a...[O, O, O, O, O, O, O, O, O, O, O, O, O, O, B-L...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, ...
11[●, Mitwirkung, im, Sinne, der, Kostenverfügung][O, O, O, O, O, O][0, 0, 0, 0, 0, 0]
22[Von, der, Ablehnung, eines, Straferlasses, fü...[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...
33[Zwar, führt, bei, der, in, §, 33, Abs., 2, TV...[O, O, O, O, O, B-REG, I-REG, I-REG, I-REG, I-...[0, 0, 0, 0, 0, 17, 17, 17, 17, 17, 17, 17, 0,...
44[Der, Wortlaut, der, Zulagenregelung, verlange...[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O][0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
\n", + "
" + ], + "text/plain": [ + " id tokens \\\n", + "0 0 [Wegen, der, Teilnahme, des, Antragstellers, a... \n", + "1 1 [●, Mitwirkung, im, Sinne, der, Kostenverfügung] \n", + "2 2 [Von, der, Ablehnung, eines, Straferlasses, fü... \n", + "3 3 [Zwar, führt, bei, der, in, §, 33, Abs., 2, TV... \n", + "4 4 [Der, Wortlaut, der, Zulagenregelung, verlange... \n", + "\n", + " ner_tags \\\n", + "0 [O, O, O, O, O, O, O, O, O, O, O, O, O, O, B-L... \n", + "1 [O, O, O, O, O, O] \n", + "2 [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ... \n", + "3 [O, O, O, O, O, B-REG, I-REG, I-REG, I-REG, I-... \n", + "4 [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O] \n", + "\n", + " fine_ner_tags \n", + "0 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, ... \n", + "1 [0, 0, 0, 0, 0, 0] \n", + "2 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... \n", + "3 [0, 0, 0, 0, 0, 17, 17, 17, 17, 17, 17, 17, 0,... \n", + "4 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] " + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [], + "source": [ + "coarse_labels_merge = {\n", + " 'B-LIT': 'Literatur', \n", + " 'B-LOC': 'Ort', \n", + " 'B-NRM': 'Norm', \n", + " 'B-ORG': 'Organisation', \n", + " 'B-PER': 'Person', \n", + " 'B-REG': 'Einzelfallregelung', \n", + " 'B-RS': 'Rechtsprechung', \n", + " 'I-LIT': 'Literatur', \n", + " 'I-LOC': 'Ort', \n", + " 'I-NRM': 'Norm', \n", + " 'I-ORG': 'Organisation', \n", + " 'I-PER': 'Person', \n", + " 'I-REG': 'Einzelfallregelung', \n", + " 'I-RS': 'Rechtsprechung', \n", + " 'O': 'O'\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "coarse_labels_merge2id = {\n", + " 'O': 0,\n", + " 'Person': 1,\n", + " 'Ort': 2,\n", + " 'Organisation': 3,\n", + " 'Norm': 4,\n", + " 'Gesetz': 5,\n", + " 'Rechtsprechung': 6,\n", + " 'Literatur': 7,\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{0: 'O',\n", + " 1: 'Person',\n", + " 2: 'Ort',\n", + " 3: 'Organisation',\n", + " 4: 'Norm',\n", + " 5: 'Einzelfallregelung',\n", + " 6: 'Rechtsprechung',\n", + " 7: 'Literatur'}" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "coarse_id2label = {v: k for k, v in coarse_labels_merge2id.items()}\n", + "coarse_id2label" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "art\n", + "building\n", + "event\n", + "location\n", + "organization\n", + "other\n", + "person\n", + "product\n" + ] + } + ], + "source": [ + "print(\"\\n\".join([(coarse_id2label.values()(lambda x: x.upper()))[1:]))" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "df[\"ner_tags\"] = df[\"ner_tags\"].apply(lambda x: [coarse_labels_merge[i] for i in x])" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idtokensner_tagsfine_ner_tags
00[Wegen, der, Teilnahme, des, Antragstellers, a...[O, O, O, O, O, O, O, O, O, O, O, O, O, O, Ort...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, ...
11[●, Mitwirkung, im, Sinne, der, Kostenverfügung][O, O, O, O, O, O][0, 0, 0, 0, 0, 0]
22[Von, der, Ablehnung, eines, Straferlasses, fü...[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...
33[Zwar, führt, bei, der, in, §, 33, Abs., 2, TV...[O, O, O, O, O, Einzelfallregelung, Einzelfall...[0, 0, 0, 0, 0, 17, 17, 17, 17, 17, 17, 17, 0,...
44[Der, Wortlaut, der, Zulagenregelung, verlange...[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O][0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
\n", + "
" + ], + "text/plain": [ + " id tokens \\\n", + "0 0 [Wegen, der, Teilnahme, des, Antragstellers, a... \n", + "1 1 [●, Mitwirkung, im, Sinne, der, Kostenverfügung] \n", + "2 2 [Von, der, Ablehnung, eines, Straferlasses, fü... \n", + "3 3 [Zwar, führt, bei, der, in, §, 33, Abs., 2, TV... \n", + "4 4 [Der, Wortlaut, der, Zulagenregelung, verlange... \n", + "\n", + " ner_tags \\\n", + "0 [O, O, O, O, O, O, O, O, O, O, O, O, O, O, Ort... \n", + "1 [O, O, O, O, O, O] \n", + "2 [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ... \n", + "3 [O, O, O, O, O, Einzelfallregelung, Einzelfall... \n", + "4 [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O] \n", + "\n", + " fine_ner_tags \n", + "0 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, ... \n", + "1 [0, 0, 0, 0, 0, 0] \n", + "2 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... \n", + "3 [0, 0, 0, 0, 0, 17, 17, 17, 17, 17, 17, 17, 0,... \n", + "4 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] " + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()\n" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "df[\"ner_tags\"] = df[\"ner_tags\"].apply(lambda x: [coarse_labels_merge2id[i] for i in x])" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idtokensner_tagsfine_ner_tags
00[Wegen, der, Teilnahme, des, Antragstellers, a...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, ...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, ...
11[●, Mitwirkung, im, Sinne, der, Kostenverfügung][0, 0, 0, 0, 0, 0][0, 0, 0, 0, 0, 0]
22[Von, der, Ablehnung, eines, Straferlasses, fü...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...
33[Zwar, führt, bei, der, in, §, 33, Abs., 2, TV...[0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, ...[0, 0, 0, 0, 0, 17, 17, 17, 17, 17, 17, 17, 0,...
44[Der, Wortlaut, der, Zulagenregelung, verlange...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0][0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
\n", + "
" + ], + "text/plain": [ + " id tokens \\\n", + "0 0 [Wegen, der, Teilnahme, des, Antragstellers, a... \n", + "1 1 [●, Mitwirkung, im, Sinne, der, Kostenverfügung] \n", + "2 2 [Von, der, Ablehnung, eines, Straferlasses, fü... \n", + "3 3 [Zwar, führt, bei, der, in, §, 33, Abs., 2, TV... \n", + "4 4 [Der, Wortlaut, der, Zulagenregelung, verlange... \n", + "\n", + " ner_tags \\\n", + "0 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, ... \n", + "1 [0, 0, 0, 0, 0, 0] \n", + "2 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... \n", + "3 [0, 0, 0, 0, 0, 5, 5, 5, 5, 5, 5, 5, 0, 0, 0, ... \n", + "4 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] \n", + "\n", + " fine_ner_tags \n", + "0 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, ... \n", + "1 [0, 0, 0, 0, 0, 0] \n", + "2 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... \n", + "3 [0, 0, 0, 0, 0, 17, 17, 17, 17, 17, 17, 17, 0,... \n", + "4 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] " + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [], + "source": [ + "df.to_parquet(\"german_ler_test.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/benchmarks/datasets/german-quotations/.gitignore b/benchmarks/datasets/german-quotations/.gitignore new file mode 100644 index 000000000..808d0e52b --- /dev/null +++ b/benchmarks/datasets/german-quotations/.gitignore @@ -0,0 +1,4 @@ +* +!.gitignore +!README.md +!quotation_attribution_creation.ipynb diff --git a/benchmarks/datasets/german-quotations/README.md b/benchmarks/datasets/german-quotations/README.md new file mode 100644 index 000000000..5bed9854d --- /dev/null +++ b/benchmarks/datasets/german-quotations/README.md @@ -0,0 +1,82 @@ +# German Quotations (Benchmark Dataset) + +## What Is This Dataset About? + +This dataset focuses on quotation attribution in German news text. + +In our benchmark setup, we transform document-level annotations into token-level span labels to support span classification for quote/speaker-related tagging. + +## Where Can It Be Found? + +- Project repository: + - https://github.com/uhh-lt/german-news-quotation-attribution-2024 + +## Links (Website / Download / Citation) + +- ACL Anthology paper: + - https://aclanthology.org/2024.lrec-main.394.pdf +- Project repository: + - https://github.com/uhh-lt/german-news-quotation-attribution-2024 + +## Benchmark Task Usage + +- Task 3: Span Classification + +## Dataset Size (Current Files) + +- `german_quotations_test.parquet`: 998 samples +- `german_direct_quotations.parquet`: 434 samples + +## How We Preprocess It + +Preprocessing is implemented in `quotation_attribution_creation.ipynb`. + +Main steps: + +1. Iterate all `.pretty.json` files from `train/`, `dev/`, and `test/`. +2. Flatten token streams from each document. +3. Derive token-level labels for quote/speaker attribution. +4. Build sequence tag arrays per document. +5. Track documents without quote annotations (`isempty`). +6. Build: + - a test evaluation file + - a larger training-oriented file that keeps all non-empty docs plus a sampled subset of empty docs +7. Export Parquet outputs. + +## Final Dataset Structure + +### File: `german_quotations_test.parquet` + +- `tokens`: token sequence +- `tags`: token-level label sequence + +### File: `german_direct_quotations.parquet` + +- `tokens`: token sequence +- `tags`: token-level label sequence +- `isempty`: whether no quote span is present in the sample +- `__index_level_0__`: pandas index artifact + +## Available Tags In The Final Files + +### `german_quotations_test.parquet` + +Observed string tag values in `tags`: + +- `O` +- `speaker` +- `quote` + +### `german_direct_quotations.parquet` + +Observed integer tag values in `tags`: + +- `0` +- `1` +- `2` + +Coarse mapping used in preprocessing: + +- `0` -> `O` +- `1` -> `speaker` +- `2` -> `quote` diff --git a/benchmarks/datasets/german-quotations/quotation_attribution_creation.ipynb b/benchmarks/datasets/german-quotations/quotation_attribution_creation.ipynb new file mode 100644 index 000000000..ec86c10c8 --- /dev/null +++ b/benchmarks/datasets/german-quotations/quotation_attribution_creation.ipynb @@ -0,0 +1,2623 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import json\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "metadata": {}, + "outputs": [], + "source": [ + "dataset_path = Path(\"datasets/german-quotations/test\")\n", + "\n", + "paths = [\n", + " Path(\"datasets/german-quotations/test\"),\n", + " Path(\"datasets/german-quotations/train\"),\n", + " Path(\"datasets/german-quotations/dev\")\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Processing datasets/german-quotations/test/83133 - DFB-Frauenteam siegt glücklich gegen Spanien (2019-06-13).pretty.json\n", + "83133 - DFB-Frauenteam siegt glücklich gegen Spanien (2019-06-13)\n", + "Processing datasets/german-quotations/test/3560 - 1,3 Millionen Bundesbürger in Deutschland haben kostenlos Pakete verschickt (2005-02-27).pretty.json\n", + "3560 - 1,3 Millionen Bundesbürger in Deutschland haben kostenlos Pakete verschickt (2005-02-27)\n", + "Processing datasets/german-quotations/test/8684 - Drogenvergehen: Deutsche nach drei Jahren Haft in Singapur wieder frei (2005-07-15).pretty.json\n", + "8684 - Drogenvergehen: Deutsche nach drei Jahren Haft in Singapur wieder frei (2005-07-15)\n", + "Processing datasets/german-quotations/test/37225 - Fieber-Epidemie in Kenia tötet mindestens 90 Menschen (2007-01-15).pretty.json\n", + "37225 - Fieber-Epidemie in Kenia tötet mindestens 90 Menschen (2007-01-15)\n", + "Processing datasets/german-quotations/test/18451 - Massenkarambolage auf der Autobahn bei Darmstadt (2005-12-26).pretty.json\n", + "18451 - Massenkarambolage auf der Autobahn bei Darmstadt (2005-12-26)\n", + "Processing datasets/german-quotations/test/39628 - Iran-Krise spitzt sich zu: Großbritannien „friert“ Beziehungen zum Iran „ein“ (2007-03-31).pretty.json\n", + "39628 - Iran-Krise spitzt sich zu: Großbritannien „friert“ Beziehungen zum Iran „ein“ (2007-03-31)\n", + "Processing datasets/german-quotations/test/12474 - Jenaer Westbahnhof wird saniert (2005-09-14).pretty.json\n", + "12474 - Jenaer Westbahnhof wird saniert (2005-09-14)\n", + "Processing datasets/german-quotations/test/58004 - Amtsinhaber Thomas Horn gewinnt Bürgermeisterwahl in Kelkheim (2009-06-25).pretty.json\n", + "58004 - Amtsinhaber Thomas Horn gewinnt Bürgermeisterwahl in Kelkheim (2009-06-25)\n", + "Processing datasets/german-quotations/test/71466 - Hoteliers rufen aus Protest gegen HRS zur HRS-freien Woche auf (2012-01-03).pretty.json\n", + "71466 - Hoteliers rufen aus Protest gegen HRS zur HRS-freien Woche auf (2012-01-03)\n", + "Processing datasets/german-quotations/test/67830 - München: Haftentlassener schlägt 82-jährigen Rentner nieder (2011-04-28).pretty.json\n", + "67830 - München: Haftentlassener schlägt 82-jährigen Rentner nieder (2011-04-28)\n", + "Processing datasets/german-quotations/test/22916 - STANDARD LIFE Versicherung geht weiter in Richtung Demutualisierung (2006-03-02).pretty.json\n", + "22916 - STANDARD LIFE Versicherung geht weiter in Richtung Demutualisierung (2006-03-02)\n", + "Processing datasets/german-quotations/test/58940 - NPD-Hetzschrift in Parteizentrale sichergestellt (2009-09-24).pretty.json\n", + "58940 - NPD-Hetzschrift in Parteizentrale sichergestellt (2009-09-24)\n", + "Processing datasets/german-quotations/test/75013 - Zwei Tote bei Massenkarambolage auf der Bundesautobahn 6 (2013-02-20).pretty.json\n", + "75013 - Zwei Tote bei Massenkarambolage auf der Bundesautobahn 6 (2013-02-20)\n", + "Processing datasets/german-quotations/test/13851 - Vogelgrippe in Europa: Kommt eine Pandemie? (2005-10-13).pretty.json\n", + "13851 - Vogelgrippe in Europa: Kommt eine Pandemie? (2005-10-13)\n", + "Processing datasets/german-quotations/test/60408 - Neue Bankenpleite in den USA (2010-01-09).pretty.json\n", + "60408 - Neue Bankenpleite in den USA (2010-01-09)\n", + "Processing datasets/german-quotations/test/67356 - Dalai Lama besorgt um versiegende Wasserquellen in der tibetischen Hochebene (2011-04-03).pretty.json\n", + "67356 - Dalai Lama besorgt um versiegende Wasserquellen in der tibetischen Hochebene (2011-04-03)\n", + "Processing datasets/german-quotations/test/63051 - Bombenleger von Viernheim muss zwölf Jahre hinter Gitter (2010-06-23).pretty.json\n", + "63051 - Bombenleger von Viernheim muss zwölf Jahre hinter Gitter (2010-06-23)\n", + "Processing datasets/german-quotations/test/75206 - 9. Markt für Dresdner Geschichte und Geschichten: Aus Vororten werden Vorstädte (2013-03-24).pretty.json\n", + "75206 - 9. Markt für Dresdner Geschichte und Geschichten: Aus Vororten werden Vorstädte (2013-03-24)\n", + "Processing datasets/german-quotations/test/15867 - Fußballprofi Castro entschied sich für Deutschland (2005-11-10).pretty.json\n", + "15867 - Fußballprofi Castro entschied sich für Deutschland (2005-11-10)\n", + "Processing datasets/german-quotations/test/42295 - Klausjürgen Wussow ist tot (2007-06-20).pretty.json\n", + "42295 - Klausjürgen Wussow ist tot (2007-06-20)\n", + "Processing datasets/german-quotations/test/16561 - Bundesregierung verteidigt Haushaltsplan 2006 (2005-12-01).pretty.json\n", + "16561 - Bundesregierung verteidigt Haushaltsplan 2006 (2005-12-01)\n", + "Processing datasets/german-quotations/test/67548 - New York: Ehemaliger Boss der La Cosa Nostra packt vor Gericht aus (2011-04-16).pretty.json\n", + "67548 - New York: Ehemaliger Boss der La Cosa Nostra packt vor Gericht aus (2011-04-16)\n", + "Processing datasets/german-quotations/test/46269 - Lebende Legende der Kurven - Oscar Niemeyer feiert seinen 100sten (2007-12-14).pretty.json\n", + "46269 - Lebende Legende der Kurven - Oscar Niemeyer feiert seinen 100sten (2007-12-14)\n", + "Processing datasets/german-quotations/test/9379 - Antonow 12 fährt leer über das Rollfeld: Bruchlandung (2005-07-27).pretty.json\n", + "9379 - Antonow 12 fährt leer über das Rollfeld: Bruchlandung (2005-07-27)\n", + "Processing datasets/german-quotations/test/65671 - Neue tunesische „Regierung der nationalen Einheit“ bereits nach einem Tag in der Krise (2011-01-18).pretty.json\n", + "65671 - Neue tunesische „Regierung der nationalen Einheit“ bereits nach einem Tag in der Krise (2011-01-18)\n", + "Processing datasets/german-quotations/test/65602 - Tunesischer Präsident Ben Ali nach Saudi-Arabien geflohen (2011-01-15).pretty.json\n", + "65602 - Tunesischer Präsident Ben Ali nach Saudi-Arabien geflohen (2011-01-15)\n", + "Processing datasets/german-quotations/test/18862 - Bad Reichenhall: Dach der Eissporthalle zusammengebrochen (2006-01-02).pretty.json\n", + "18862 - Bad Reichenhall: Dach der Eissporthalle zusammengebrochen (2006-01-02)\n", + "Processing datasets/german-quotations/test/54074 - Angeklagte nach 31 Jahre zurückliegendem Mord freigesprochen (2008-11-16).pretty.json\n", + "54074 - Angeklagte nach 31 Jahre zurückliegendem Mord freigesprochen (2008-11-16)\n", + "Processing datasets/german-quotations/test/72243 - Arbeitslosigkeit in der Eurozone erreicht Rekordniveau (2012-04-13).pretty.json\n", + "72243 - Arbeitslosigkeit in der Eurozone erreicht Rekordniveau (2012-04-13)\n", + "Processing datasets/german-quotations/test/4249 - Müntefering will Pflegeversicherungspflicht ausweiten (2005-03-26).pretty.json\n", + "4249 - Müntefering will Pflegeversicherungspflicht ausweiten (2005-03-26)\n", + "Processing datasets/german-quotations/test/53341 - Videospiel: Fallout 3 mit speziell für den Trailer angefertigten, hochauflösendem Inhalt (2008-10-07).pretty.json\n", + "53341 - Videospiel: Fallout 3 mit speziell für den Trailer angefertigten, hochauflösendem Inhalt (2008-10-07)\n", + "Processing datasets/german-quotations/test/43325 - Koblenz: Zwei Erdbeben am Freitagmorgen (2007-08-03).pretty.json\n", + "43325 - Koblenz: Zwei Erdbeben am Freitagmorgen (2007-08-03)\n", + "Processing datasets/german-quotations/test/73320 - Neuwied: Drei Wochen altes in Tschechien entführtes Mädchen aufgefunden (2012-07-10).pretty.json\n", + "73320 - Neuwied: Drei Wochen altes in Tschechien entführtes Mädchen aufgefunden (2012-07-10)\n", + "Processing datasets/german-quotations/test/65177 - Venezuela: Präsident Chávez regiert per Dekret (2010-12-18).pretty.json\n", + "65177 - Venezuela: Präsident Chávez regiert per Dekret (2010-12-18)\n", + "Processing datasets/german-quotations/test/83299 - Kamerun bekämpft Rekrutierungen durch Boko Haram mit Ziegen und Schafen (2019-07-14).pretty.json\n", + "83299 - Kamerun bekämpft Rekrutierungen durch Boko Haram mit Ziegen und Schafen (2019-07-14)\n", + "Processing datasets/german-quotations/test/63594 - Abrissarbeiten für neuen Stuttgarter Bahnhof beginnen unter Protest (2010-08-28).pretty.json\n", + "63594 - Abrissarbeiten für neuen Stuttgarter Bahnhof beginnen unter Protest (2010-08-28)\n", + "Processing datasets/german-quotations/test/66750 - US-Luftwaffe schießt unbemanntes Raumschiff ins All (2011-03-06).pretty.json\n", + "66750 - US-Luftwaffe schießt unbemanntes Raumschiff ins All (2011-03-06)\n", + "Processing datasets/german-quotations/test/74583 - Skandal um Steuerfahndung in Griechenland (2013-01-01).pretty.json\n", + "74583 - Skandal um Steuerfahndung in Griechenland (2013-01-01)\n", + "Processing datasets/german-quotations/test/37853 - Iran: Angeblich neues Heilmittel für AIDS entdeckt (2007-02-06).pretty.json\n", + "37853 - Iran: Angeblich neues Heilmittel für AIDS entdeckt (2007-02-06)\n", + "Processing datasets/german-quotations/test/34748 - Großaufgebot von rund 2.500 Polizisten schützte NPD-Demo in Bremen (2006-11-04).pretty.json\n", + "34748 - Großaufgebot von rund 2.500 Polizisten schützte NPD-Demo in Bremen (2006-11-04)\n", + "Processing datasets/german-quotations/test/62849 - Köhler-Nachfolge: Kandidaten von Schwarz-Gelb und Rot-Grün stehen fest (2010-06-05).pretty.json\n", + "62849 - Köhler-Nachfolge: Kandidaten von Schwarz-Gelb und Rot-Grün stehen fest (2010-06-05)\n", + "Processing datasets/german-quotations/test/44870 - Methodos e.V. - Freiburger Schüler lernen für das Abitur im eigenen Verein (2007-10-16).pretty.json\n", + "44870 - Methodos e.V. - Freiburger Schüler lernen für das Abitur im eigenen Verein (2007-10-16)\n", + "Processing datasets/german-quotations/test/69083 - Mafiamorde von Duisburg: Italienisches Gericht spricht lebenslange Haftstrafe aus (2011-07-16).pretty.json\n", + "69083 - Mafiamorde von Duisburg: Italienisches Gericht spricht lebenslange Haftstrafe aus (2011-07-16)\n", + "Processing datasets/german-quotations/test/6226 - CDU und CSU streiten über künftigen Kurs in der Sozialpolitik (2005-06-14).pretty.json\n", + "6226 - CDU und CSU streiten über künftigen Kurs in der Sozialpolitik (2005-06-14)\n", + "Processing datasets/german-quotations/test/79481 - Widerstand in Polen gegen die Aufnahme von Flüchtlingen (2016-02-07).pretty.json\n", + "79481 - Widerstand in Polen gegen die Aufnahme von Flüchtlingen (2016-02-07)\n", + "Processing datasets/german-quotations/test/57479 - Pädagogische Hochschule Heidelberg verhängt Haushaltssperre (2009-05-21).pretty.json\n", + "57479 - Pädagogische Hochschule Heidelberg verhängt Haushaltssperre (2009-05-21)\n", + "Processing datasets/german-quotations/test/71380 - München: Angriff mit Samuraischwert (2012-01-25).pretty.json\n", + "71380 - München: Angriff mit Samuraischwert (2012-01-25)\n", + "Processing datasets/german-quotations/test/43568 - Bombenanschlag auf russischen Schnellzug (2007-08-14).pretty.json\n", + "43568 - Bombenanschlag auf russischen Schnellzug (2007-08-14)\n", + "Processing datasets/german-quotations/test/4927 - Skelett aus Altdorf stammt von Steinzeitmann (2005-04-27).pretty.json\n", + "4927 - Skelett aus Altdorf stammt von Steinzeitmann (2005-04-27)\n", + "Processing datasets/german-quotations/test/51401 - Nelson Mandela wird 90 (2008-06-28).pretty.json\n", + "51401 - Nelson Mandela wird 90 (2008-06-28)\n", + "Processing datasets/german-quotations/test/68250 - Afghanistan: Bundeswehrsoldaten schossen gezielt auf Demonstranten (2011-05-21).pretty.json\n", + "68250 - Afghanistan: Bundeswehrsoldaten schossen gezielt auf Demonstranten (2011-05-21)\n", + "Processing datasets/german-quotations/test/11014 - Messdiener auf dem Weltjugendtag (2005-08-19).pretty.json\n", + "11014 - Messdiener auf dem Weltjugendtag (2005-08-19)\n", + "Processing datasets/german-quotations/test/26748 - Edmund Stoiber unterstützt erneute Olympia-Kandidatur Münchens (2006-05-13).pretty.json\n", + "26748 - Edmund Stoiber unterstützt erneute Olympia-Kandidatur Münchens (2006-05-13)\n", + "Processing datasets/german-quotations/test/60335 - Obama: Keine weiteren Guantánamo-Häftlinge in den Jemen (2010-01-06).pretty.json\n", + "60335 - Obama: Keine weiteren Guantánamo-Häftlinge in den Jemen (2010-01-06)\n", + "Processing datasets/german-quotations/test/77466 - Peking: „Trauer“ über die Vorgänge in Hong Kong (2014-09-29).pretty.json\n", + "77466 - Peking: „Trauer“ über die Vorgänge in Hong Kong (2014-09-29)\n", + "Processing datasets/german-quotations/test/12299 - Potentiell gefährliche Lücke im Internet-Browser „Firefox“ (2005-09-10).pretty.json\n", + "12299 - Potentiell gefährliche Lücke im Internet-Browser „Firefox“ (2005-09-10)\n", + "Processing datasets/german-quotations/test/9835 - Inselrepublik Kap Verde erhält eine eigene Börse (2005-08-04).pretty.json\n", + "9835 - Inselrepublik Kap Verde erhält eine eigene Börse (2005-08-04)\n", + "Processing datasets/german-quotations/test/56411 - US-Autoabsatz bricht im Februar 2009 dramatisch ein (2009-03-09).pretty.json\n", + "56411 - US-Autoabsatz bricht im Februar 2009 dramatisch ein (2009-03-09)\n", + "Processing datasets/german-quotations/test/56312 - Millionenschaden bei Caravan-Händler Fassbender in Oldenburg (2009-02-28).pretty.json\n", + "56312 - Millionenschaden bei Caravan-Händler Fassbender in Oldenburg (2009-02-28)\n", + "Processing datasets/german-quotations/test/18120 - IWF: Schuldenerlass für die ärmsten Länder der Erde (2005-12-23).pretty.json\n", + "18120 - IWF: Schuldenerlass für die ärmsten Länder der Erde (2005-12-23)\n", + "Processing datasets/german-quotations/test/41916 - G8-Proteste: Anzahl schwer verletzter Polizisten möglicherweise als zu hoch angegeben (2007-06-06).pretty.json\n", + "41916 - G8-Proteste: Anzahl schwer verletzter Polizisten möglicherweise als zu hoch angegeben (2007-06-06)\n", + "Processing datasets/german-quotations/test/74940 - Ukraine: Tote bei Flugzeugunglück (2013-02-14).pretty.json\n", + "74940 - Ukraine: Tote bei Flugzeugunglück (2013-02-14)\n", + "Processing datasets/german-quotations/test/59230 - Falschmeldung von einem Rücktritt Wolfgang Schäubles auf einer CDU-Website (2009-10-07).pretty.json\n", + "59230 - Falschmeldung von einem Rücktritt Wolfgang Schäubles auf einer CDU-Website (2009-10-07)\n", + "Processing datasets/german-quotations/test/12825 - Wohnungsdurchsuchung im Fall des eBay-Babys (2005-09-23).pretty.json\n", + "12825 - Wohnungsdurchsuchung im Fall des eBay-Babys (2005-09-23)\n", + "Processing datasets/german-quotations/test/84324 - Spannungen an der sudanesisch-äthiopischen Grenze in der Konfliktregion al-Fashqa nehmen zu (2021-01-22).pretty.json\n", + "84324 - Spannungen an der sudanesisch-äthiopischen Grenze in der Konfliktregion al-Fashqa nehmen zu (2021-01-22)\n", + "Processing datasets/german-quotations/test/37332 - Opposition: Parlament muss über Tornado-Einsatz entscheiden (2007-01-21).pretty.json\n", + "37332 - Opposition: Parlament muss über Tornado-Einsatz entscheiden (2007-01-21)\n", + "Processing datasets/german-quotations/test/73588 - London: Erste Olympia-Medaillen für Deutschland erkämpft (2012-07-31).pretty.json\n", + "73588 - London: Erste Olympia-Medaillen für Deutschland erkämpft (2012-07-31)\n", + "Processing datasets/german-quotations/test/23777 - Bundesregierung stimmt Kompromiss für EU-Führerschein zu (2006-03-17).pretty.json\n", + "23777 - Bundesregierung stimmt Kompromiss für EU-Führerschein zu (2006-03-17)\n", + "Processing datasets/german-quotations/test/76513 - Postume Begnadigung für Alan Turing (2013-12-25).pretty.json\n", + "76513 - Postume Begnadigung für Alan Turing (2013-12-25)\n", + "Processing datasets/german-quotations/test/27244 - Popstar Madonna startete neue Tournee (2006-05-25).pretty.json\n", + "27244 - Popstar Madonna startete neue Tournee (2006-05-25)\n", + "Processing datasets/german-quotations/test/29410 - Schlägerei an Bord eines serbischen Flugzeuges (2006-07-02).pretty.json\n", + "29410 - Schlägerei an Bord eines serbischen Flugzeuges (2006-07-02)\n", + "Processing datasets/german-quotations/test/50115 - 13. Karneval der Kulturen in Berlin (2008-05-12).pretty.json\n", + "50115 - 13. Karneval der Kulturen in Berlin (2008-05-12)\n", + "Processing datasets/german-quotations/test/63379 - Brite schreitet als erster Mensch den Amazonas ab (2010-08-10).pretty.json\n", + "63379 - Brite schreitet als erster Mensch den Amazonas ab (2010-08-10)\n", + "Processing datasets/german-quotations/test/2954 - In mehreren deutschen Städten demonstrieren StudentInnen gegen Studiengebühren (2005-02-03).pretty.json\n", + "2954 - In mehreren deutschen Städten demonstrieren StudentInnen gegen Studiengebühren (2005-02-03)\n", + "Processing datasets/german-quotations/test/8648 - Computerhersteller Apple gibt seine Quartalszahlen bekannt (2005-07-15).pretty.json\n", + "8648 - Computerhersteller Apple gibt seine Quartalszahlen bekannt (2005-07-15)\n", + "Processing datasets/german-quotations/test/83285 - 22 neue Stolpersteine in Hannover (2019-07-12).pretty.json\n", + "83285 - 22 neue Stolpersteine in Hannover (2019-07-12)\n", + "Processing datasets/german-quotations/test/31608 - Hamas und Al-Fatah bilden Regierung der Nationalen Einheit (2006-09-14).pretty.json\n", + "31608 - Hamas und Al-Fatah bilden Regierung der Nationalen Einheit (2006-09-14)\n", + "Processing datasets/german-quotations/test/30393 - Saddam Hussein im Krankenhaus (2006-07-24).pretty.json\n", + "30393 - Saddam Hussein im Krankenhaus (2006-07-24)\n", + "Processing datasets/german-quotations/test/44492 - Raumsonde Dawn auf dem Weg zum Asteroidengürtel (2007-09-27).pretty.json\n", + "44492 - Raumsonde Dawn auf dem Weg zum Asteroidengürtel (2007-09-27)\n", + "Processing datasets/german-quotations/test/72691 - Indien: Polizei stoppt Eltern beim Versuch, Säugling lebendig zu begraben (2012-05-14).pretty.json\n", + "72691 - Indien: Polizei stoppt Eltern beim Versuch, Säugling lebendig zu begraben (2012-05-14)\n", + "Processing datasets/german-quotations/test/8417 - Stufenplan für Europas Raumfähre „Kliper“ (2005-07-11).pretty.json\n", + "8417 - Stufenplan für Europas Raumfähre „Kliper“ (2005-07-11)\n", + "Processing datasets/german-quotations/test/70697 - 13-jährige Tochter in Stolzenau vom Vater erschossen (2011-12-06).pretty.json\n", + "70697 - 13-jährige Tochter in Stolzenau vom Vater erschossen (2011-12-06)\n", + "Processing datasets/german-quotations/test/23505 - Weiterer Meilenstein bei Wikipedia (2006-03-12).pretty.json\n", + "23505 - Weiterer Meilenstein bei Wikipedia (2006-03-12)\n", + "Processing datasets/german-quotations/test/14446 - Platz acht für Martina Ertl-Renz im Riesenslalom (2005-10-23).pretty.json\n", + "14446 - Platz acht für Martina Ertl-Renz im Riesenslalom (2005-10-23)\n", + "Processing datasets/german-quotations/test/50808 - Der Hamburger Schriftsteller Peter Rühmkorf starb im Alter von 78 Jahren (2008-06-09).pretty.json\n", + "50808 - Der Hamburger Schriftsteller Peter Rühmkorf starb im Alter von 78 Jahren (2008-06-09)\n", + "Processing datasets/german-quotations/test/38864 - Nach Notbremsung: 450 ICE-Passagiere müssen im Tunnel umsteigen (2007-03-07).pretty.json\n", + "38864 - Nach Notbremsung: 450 ICE-Passagiere müssen im Tunnel umsteigen (2007-03-07)\n", + "Processing datasets/german-quotations/test/37718 - FC Bayern München entlässt Trainer Magath (2007-01-31).pretty.json\n", + "37718 - FC Bayern München entlässt Trainer Magath (2007-01-31)\n", + "Processing datasets/german-quotations/test/49445 - Fußball-Wettskandal: Außergerichtliche Einigung zwischem dem DFB und dem ehemaligen Schiedsrichter Robert Hoyzer (2008-04-04).pretty.json\n", + "49445 - Fußball-Wettskandal: Außergerichtliche Einigung zwischem dem DFB und dem ehemaligen Schiedsrichter Robert Hoyzer (2008-04-04)\n", + "Processing datasets/german-quotations/test/29071 - Barbara Ludwig gewann die Oberbürgermeisterwahlen in Chemnitz (2006-06-27).pretty.json\n", + "29071 - Barbara Ludwig gewann die Oberbürgermeisterwahlen in Chemnitz (2006-06-27)\n", + "Processing datasets/german-quotations/test/4130 - Weitere 130.000 ehemalige Sozialhilfeempfänger in der Statistik (2005-03-24).pretty.json\n", + "4130 - Weitere 130.000 ehemalige Sozialhilfeempfänger in der Statistik (2005-03-24)\n", + "Processing datasets/german-quotations/test/77285 - Indisch-Pakistanische Wassergespräche ergebnislos vertagt (2014-08-29).pretty.json\n", + "77285 - Indisch-Pakistanische Wassergespräche ergebnislos vertagt (2014-08-29)\n", + "Processing datasets/german-quotations/test/74142 - USA erhebt Zölle auf billige chinesische Solarmodule (2012-10-18).pretty.json\n", + "74142 - USA erhebt Zölle auf billige chinesische Solarmodule (2012-10-18)\n", + "Processing datasets/german-quotations/test/30241 - Neue Version der 3D-Software Blender erschienen (2006-07-19).pretty.json\n", + "30241 - Neue Version der 3D-Software Blender erschienen (2006-07-19)\n", + "Processing datasets/german-quotations/test/45586 - Feldversuch in NRW für versetzt liegende Warnschwellen an Autobahnbaustellen gestartet (2007-12-03).pretty.json\n", + "45586 - Feldversuch in NRW für versetzt liegende Warnschwellen an Autobahnbaustellen gestartet (2007-12-03)\n", + "Processing datasets/german-quotations/test/8915 - Union über Einführung der neuen Rechtschreibung gespalten (2005-07-19).pretty.json\n", + "8915 - Union über Einführung der neuen Rechtschreibung gespalten (2005-07-19)\n", + "Processing datasets/german-quotations/test/12162 - Ehemaliger TV-Moderator Türck freigesprochen (2005-09-09).pretty.json\n", + "12162 - Ehemaliger TV-Moderator Türck freigesprochen (2005-09-09)\n", + "Processing datasets/german-quotations/test/8461 - Zukunft der Kassel Huskies ungewiss (2005-07-11).pretty.json\n", + "8461 - Zukunft der Kassel Huskies ungewiss (2005-07-11)\n", + "Processing datasets/german-quotations/test/55456 - Haie sind die wahren Opfer (2009-01-21).pretty.json\n", + "55456 - Haie sind die wahren Opfer (2009-01-21)\n", + "Processing datasets/german-quotations/test/72180 - US-Vorwahlen: Santorum setzt sich in Louisiana durch (2012-03-28).pretty.json\n", + "72180 - US-Vorwahlen: Santorum setzt sich in Louisiana durch (2012-03-28)\n", + "Processing datasets/german-quotations/test/59934 - Băsescu gewinnt Präsidentschafts-Stichwahl in Rumänien (2009-12-09).pretty.json\n", + "59934 - Băsescu gewinnt Präsidentschafts-Stichwahl in Rumänien (2009-12-09)\n", + "Processing datasets/german-quotations/test/10437 - Chinesischer Astronaut soll 2007 Weltraumspaziergang machen (2005-08-12).pretty.json\n", + "10437 - Chinesischer Astronaut soll 2007 Weltraumspaziergang machen (2005-08-12)\n", + "Processing datasets/german-quotations/test/4030 - EU-Stabilitätspakt wird gelockert (2005-03-21).pretty.json\n", + "4030 - EU-Stabilitätspakt wird gelockert (2005-03-21)\n", + "Processing datasets/german-quotations/test/37231 - 13-Jährige fertigte Nacktbilder selbst an (2007-01-16).pretty.json\n", + "37231 - 13-Jährige fertigte Nacktbilder selbst an (2007-01-16)\n", + "Processing datasets/german-quotations/test/27778 - Ausnahmezustand in Osttimor erklärt – Lage beruhigt sich langsam (2006-05-31).pretty.json\n", + "27778 - Ausnahmezustand in Osttimor erklärt – Lage beruhigt sich langsam (2006-05-31)\n", + "Processing datasets/german-quotations/test/63081 - Fußball-WM: Deutschlands Flop gegen Serbien (2010-06-24).pretty.json\n", + "63081 - Fußball-WM: Deutschlands Flop gegen Serbien (2010-06-24)\n", + "Processing datasets/german-quotations/test/51722 - Elektroauto „Ze-0“: In China gebaut, in Europa verkauft (2008-07-16).pretty.json\n", + "51722 - Elektroauto „Ze-0“: In China gebaut, in Europa verkauft (2008-07-16)\n", + "Processing datasets/german-quotations/test/44763 - Dumawahl: Oppositionspartei darf nicht antreten (2007-10-11).pretty.json\n", + "44763 - Dumawahl: Oppositionspartei darf nicht antreten (2007-10-11)\n", + "Processing datasets/german-quotations/test/70474 - Lebensbedrohlich Verletzter nach Überschlag bei Kaiserslautern (2011-11-21).pretty.json\n", + "70474 - Lebensbedrohlich Verletzter nach Überschlag bei Kaiserslautern (2011-11-21)\n", + "Processing datasets/german-quotations/test/71869 - Mexiko City: Die letzten VW-Käfer nehmen ihren Abschied als Taxi (2012-03-03).pretty.json\n", + "71869 - Mexiko City: Die letzten VW-Käfer nehmen ihren Abschied als Taxi (2012-03-03)\n", + "Processing datasets/german-quotations/test/33318 - ISS-Expedition 13 und Raumflugteilnehmerin Anousheh Ansari in Kasachstan gelandet (2006-09-29).pretty.json\n", + "33318 - ISS-Expedition 13 und Raumflugteilnehmerin Anousheh Ansari in Kasachstan gelandet (2006-09-29)\n", + "Processing datasets/german-quotations/test/73472 - Japan: Arbeiter im Atomkraftwerk sollten Strahlenbelastung verschleiern (2012-07-21).pretty.json\n", + "73472 - Japan: Arbeiter im Atomkraftwerk sollten Strahlenbelastung verschleiern (2012-07-21)\n", + "Processing datasets/german-quotations/test/68080 - Syrien: Artilleriebeschuss der Stadt Homs (2011-05-11).pretty.json\n", + "68080 - Syrien: Artilleriebeschuss der Stadt Homs (2011-05-11)\n", + "Processing datasets/german-quotations/test/65540 - Abschlussbericht zum Flugzeugabsturz der Maschine des polnischen Präsidenten Kaczyński vorgelegt (2011-01-12).pretty.json\n", + "65540 - Abschlussbericht zum Flugzeugabsturz der Maschine des polnischen Präsidenten Kaczyński vorgelegt (2011-01-12)\n", + "Processing datasets/german-quotations/test/26502 - Thailand: Verfassungsgericht erklärt die Wahlen vom 2. April für nichtig (2006-05-08).pretty.json\n", + "26502 - Thailand: Verfassungsgericht erklärt die Wahlen vom 2. April für nichtig (2006-05-08)\n", + "Processing datasets/german-quotations/test/46376 - Monteur stürzt im Windrad ab (2007-12-23).pretty.json\n", + "46376 - Monteur stürzt im Windrad ab (2007-12-23)\n", + "Processing datasets/german-quotations/test/81036 - Fahrverbot für Dieselfahrzeuge wird in deutschen Städten diskutiert (2016-05-04).pretty.json\n", + "81036 - Fahrverbot für Dieselfahrzeuge wird in deutschen Städten diskutiert (2016-05-04)\n", + "Processing datasets/german-quotations/test/64221 - Stiftung Warentest schlägt Alarm: Schadstoffe im Kinderspielzeug (2010-10-22).pretty.json\n", + "64221 - Stiftung Warentest schlägt Alarm: Schadstoffe im Kinderspielzeug (2010-10-22)\n", + "Processing datasets/german-quotations/test/59848 - Israel kündigt 10 Monate Siedlungsstopp in der Westbank an (2009-11-27).pretty.json\n", + "59848 - Israel kündigt 10 Monate Siedlungsstopp in der Westbank an (2009-11-27)\n", + "Processing datasets/german-quotations/test/70345 - Nach Polizistenmord in Augsburg: 45 neue Hinweise nach ZDF-Sendung (2011-11-11).pretty.json\n", + "70345 - Nach Polizistenmord in Augsburg: 45 neue Hinweise nach ZDF-Sendung (2011-11-11)\n", + "Processing datasets/german-quotations/test/16260 - Til und Dana Schweiger: Alles aus (2005-11-22).pretty.json\n", + "16260 - Til und Dana Schweiger: Alles aus (2005-11-22)\n", + "Processing datasets/german-quotations/test/6745 - Satellit „Intelsat Americas 8“ von Seeplattform aus ins All gestartet (2005-06-23).pretty.json\n", + "6745 - Satellit „Intelsat Americas 8“ von Seeplattform aus ins All gestartet (2005-06-23)\n", + "Processing datasets/german-quotations/test/72779 - Blockupy Frankfurt: Friedliche Demonstrationen gegen Kapitalismus und Krisenpolitik (2012-05-21).pretty.json\n", + "72779 - Blockupy Frankfurt: Friedliche Demonstrationen gegen Kapitalismus und Krisenpolitik (2012-05-21)\n", + "Processing datasets/german-quotations/test/27945 - Universal-Film dreht Film über Hurrikan Katrina (2006-06-04).pretty.json\n", + "27945 - Universal-Film dreht Film über Hurrikan Katrina (2006-06-04)\n", + "Processing datasets/german-quotations/test/82818 - Google-freies Android soll bald verfügbar sein (2019-02-03).pretty.json\n", + "82818 - Google-freies Android soll bald verfügbar sein (2019-02-03)\n", + "Processing datasets/german-quotations/test/52445 - Baden-Württemberg: Umstrittenes Polizeigesetz wird in den Landtag eingebracht, neues Versammlungsrecht zur Anhörung freigegeben (2008-08-29).pretty.json\n", + "52445 - Baden-Württemberg: Umstrittenes Polizeigesetz wird in den Landtag eingebracht, neues Versammlungsrecht zur Anhörung freigegeben (2008-08-29)\n", + "Processing datasets/german-quotations/test/12984 - Jakob Maria Mierscheid: Diesmal versagte auch seine Wahlprognose (2005-09-27).pretty.json\n", + "12984 - Jakob Maria Mierscheid: Diesmal versagte auch seine Wahlprognose (2005-09-27)\n", + "Processing datasets/german-quotations/test/45193 - Umstrittene Spielregeln bei Fernsehsender 9Live (2007-11-04).pretty.json\n", + "45193 - Umstrittene Spielregeln bei Fernsehsender 9Live (2007-11-04)\n", + "Processing datasets/german-quotations/test/21026 - Kreis Euskirchen: Das Konjunkturbarometer steigt (2006-01-29).pretty.json\n", + "21026 - Kreis Euskirchen: Das Konjunkturbarometer steigt (2006-01-29)\n", + "Processing datasets/german-quotations/test/27195 - Max Raabe und sein Palast Orchester auf Tournee durch China (2006-05-21).pretty.json\n", + "27195 - Max Raabe und sein Palast Orchester auf Tournee durch China (2006-05-21)\n", + "Processing datasets/german-quotations/test/11733 - Pakistan und Israel verhandeln über die Aufnahme diplomatischer Beziehungen (2005-09-01).pretty.json\n", + "11733 - Pakistan und Israel verhandeln über die Aufnahme diplomatischer Beziehungen (2005-09-01)\n", + "Processing datasets/german-quotations/test/42342 - Eiszeitliche Wolfsart fraß Mammuts und Bisons (2007-06-22).pretty.json\n", + "42342 - Eiszeitliche Wolfsart fraß Mammuts und Bisons (2007-06-22)\n", + "Processing datasets/german-quotations/test/1326 - Umbauarbeiten am Erlweinspeicher bis Frühjahr 2006 (2004-12-07).pretty.json\n", + "1326 - Umbauarbeiten am Erlweinspeicher bis Frühjahr 2006 (2004-12-07)\n", + "Processing datasets/german-quotations/test/13474 - Wieder Terroranschlag in Thailands Südprovinzen (2005-10-06).pretty.json\n", + "13474 - Wieder Terroranschlag in Thailands Südprovinzen (2005-10-06)\n", + "Processing datasets/german-quotations/test/18703 - Kairo: Zehn sudanesische Flüchtlinge bei der Erstürmung ihres Lagers getötet (2005-12-30).pretty.json\n", + "18703 - Kairo: Zehn sudanesische Flüchtlinge bei der Erstürmung ihres Lagers getötet (2005-12-30)\n", + "Processing datasets/german-quotations/test/62091 - Deutscher Bundesverkehrsminister will vorläufig keine staatlichen Kaufhilfen für Elektroautos (2010-04-12).pretty.json\n", + "62091 - Deutscher Bundesverkehrsminister will vorläufig keine staatlichen Kaufhilfen für Elektroautos (2010-04-12)\n", + "Processing datasets/german-quotations/test/13406 - Mit dem Heißluftballon gegen Kinderarbeit (2005-10-04).pretty.json\n", + "13406 - Mit dem Heißluftballon gegen Kinderarbeit (2005-10-04)\n", + "Processing datasets/german-quotations/test/72379 - Deutschland: Salafisten wollen Gratis-Koran verteilen (2012-04-14).pretty.json\n", + "72379 - Deutschland: Salafisten wollen Gratis-Koran verteilen (2012-04-14)\n", + "Processing datasets/german-quotations/test/20914 - Auslosung der Ausscheidungsspiele der Europameisterschaft 2008 stattgefunden (2006-01-28).pretty.json\n", + "20914 - Auslosung der Ausscheidungsspiele der Europameisterschaft 2008 stattgefunden (2006-01-28)\n", + "Processing datasets/german-quotations/test/34097 - Beinahe Flugzeug-Zusammenstoß über Rheinland-Pfalz (2006-10-23).pretty.json\n", + "34097 - Beinahe Flugzeug-Zusammenstoß über Rheinland-Pfalz (2006-10-23)\n", + "Processing datasets/german-quotations/test/746 - SS-Lagerkommandant Schwammberger gestorben (2004-12-03).pretty.json\n", + "746 - SS-Lagerkommandant Schwammberger gestorben (2004-12-03)\n", + "Processing datasets/german-quotations/test/10546 - Deutsche Speerwerferinnen gewinnen in Helsinki Silber- und Bronzemedaille (2005-08-14).pretty.json\n", + "10546 - Deutsche Speerwerferinnen gewinnen in Helsinki Silber- und Bronzemedaille (2005-08-14)\n", + "Processing datasets/german-quotations/test/33349 - Campus Symposium 2006 (2006-10-03).pretty.json\n", + "33349 - Campus Symposium 2006 (2006-10-03)\n", + "Processing datasets/german-quotations/test/80411 - Brüssel: Drei Festnahmen bei Anti-Terror-Einsatz (2017-01-15).pretty.json\n", + "80411 - Brüssel: Drei Festnahmen bei Anti-Terror-Einsatz (2017-01-15)\n", + "Processing datasets/german-quotations/test/5080 - IIHF Eishockey WM in Wien und Innsbruck eröffnet (2005-05-01).pretty.json\n", + "5080 - IIHF Eishockey WM in Wien und Innsbruck eröffnet (2005-05-01)\n", + "Processing datasets/german-quotations/test/45080 - BGH kippt Haftbefehl gegen Andrej H. (2007-10-24).pretty.json\n", + "45080 - BGH kippt Haftbefehl gegen Andrej H. (2007-10-24)\n", + "Processing datasets/german-quotations/test/64021 - Myanmar: Aung San Suu Kyi wird bei den Nationalwahlen nicht abstimmen (2010-10-13).pretty.json\n", + "64021 - Myanmar: Aung San Suu Kyi wird bei den Nationalwahlen nicht abstimmen (2010-10-13)\n", + "Processing datasets/german-quotations/test/20690 - Notlandung einer Boeing 747 auf dem Lütticher Flughafen (2006-01-24).pretty.json\n", + "20690 - Notlandung einer Boeing 747 auf dem Lütticher Flughafen (2006-01-24)\n", + "Processing datasets/german-quotations/test/32884 - Gefahrgutunfall auf der A3 bei Limburg (2006-09-15).pretty.json\n", + "32884 - Gefahrgutunfall auf der A3 bei Limburg (2006-09-15)\n", + "Processing datasets/german-quotations/train/4174 - Extreme Infektionsgefahr in Krankenhäusern der „Dritten Welt“ (2005-03-25).pretty.json\n", + "4174 - Extreme Infektionsgefahr in Krankenhäusern der „Dritten Welt“ (2005-03-25)\n", + "Processing datasets/german-quotations/train/84297 - Indonesien: Boeing 737-500 vermutlich abgestürzt (2021-01-10).pretty.json\n", + "84297 - Indonesien: Boeing 737-500 vermutlich abgestürzt (2021-01-10)\n", + "Processing datasets/german-quotations/train/81009 - Wikipedia in der Türkei nach behördlicher Anordnung blockiert (2017-04-29).pretty.json\n", + "81009 - Wikipedia in der Türkei nach behördlicher Anordnung blockiert (2017-04-29)\n", + "Processing datasets/german-quotations/train/42267 - Umgestürzter Holz-Transporter blockiert Sauerlandlinie (2007-06-25).pretty.json\n", + "42267 - Umgestürzter Holz-Transporter blockiert Sauerlandlinie (2007-06-25)\n", + "Processing datasets/german-quotations/train/22065 - Oslo: Prozessbeginn um Raub der Munch-Gemälde (2006-02-14).pretty.json\n", + "22065 - Oslo: Prozessbeginn um Raub der Munch-Gemälde (2006-02-14)\n", + "Processing datasets/german-quotations/train/82396 - Busunfall im Colca-Tal: zwei deutsche Touristen ums Leben gekommen (2018-04-21).pretty.json\n", + "82396 - Busunfall im Colca-Tal: zwei deutsche Touristen ums Leben gekommen (2018-04-21)\n", + "Processing datasets/german-quotations/train/16935 - Teheran: Maschine war vor Absturz defekt (2005-12-07).pretty.json\n", + "16935 - Teheran: Maschine war vor Absturz defekt (2005-12-07)\n", + "Processing datasets/german-quotations/train/61683 - Indonesien: Meistgesuchter Terrorist Südostasiens anscheinend getötet (2010-03-10).pretty.json\n", + "61683 - Indonesien: Meistgesuchter Terrorist Südostasiens anscheinend getötet (2010-03-10)\n", + "Processing datasets/german-quotations/train/72501 - Frankreich: Wahllokale nach erster Runde der Präsidentenwahl geschlossen (2012-04-22).pretty.json\n", + "72501 - Frankreich: Wahllokale nach erster Runde der Präsidentenwahl geschlossen (2012-04-22)\n", + "Processing datasets/german-quotations/train/6103 - Brand im Wiener Prater (2005-06-11).pretty.json\n", + "6103 - Brand im Wiener Prater (2005-06-11)\n", + "Processing datasets/german-quotations/train/45882 - Aufwind am deutschen Arbeitsmarkt hält an – Arbeitslosenquote auf niedrigstem Stand seit 1993 (2007-11-29).pretty.json\n", + "45882 - Aufwind am deutschen Arbeitsmarkt hält an – Arbeitslosenquote auf niedrigstem Stand seit 1993 (2007-11-29)\n", + "Processing datasets/german-quotations/train/18625 - Bundespräsident Köhler sieht „wachsende Kluft zwischen Arm und Reich“ (2005-12-29).pretty.json\n", + "18625 - Bundespräsident Köhler sieht „wachsende Kluft zwischen Arm und Reich“ (2005-12-29)\n", + "Processing datasets/german-quotations/train/33184 - NASA-Chef Michael Griffin besucht China (2006-09-25).pretty.json\n", + "33184 - NASA-Chef Michael Griffin besucht China (2006-09-25)\n", + "Processing datasets/german-quotations/train/37935 - Historischer Bauernhof brennt bis auf die Grundmauern ab (2007-02-07).pretty.json\n", + "37935 - Historischer Bauernhof brennt bis auf die Grundmauern ab (2007-02-07)\n", + "Processing datasets/german-quotations/train/55776 - Die größte Riesenschlange aller Zeiten war 13 Meter lang (2009-02-05).pretty.json\n", + "55776 - Die größte Riesenschlange aller Zeiten war 13 Meter lang (2009-02-05)\n", + "Processing datasets/german-quotations/train/22886 - Schwere Ausschreitungen in der Republik Irland (2006-02-27).pretty.json\n", + "22886 - Schwere Ausschreitungen in der Republik Irland (2006-02-27)\n", + "Processing datasets/german-quotations/train/64651 - Verhandlungen zu EU-Haushalt vorerst gescheitert (2010-11-16).pretty.json\n", + "64651 - Verhandlungen zu EU-Haushalt vorerst gescheitert (2010-11-16)\n", + "Processing datasets/german-quotations/train/50033 - Italienische Polizei führte Schlag gegen die ’Ndrangheta (2008-05-09).pretty.json\n", + "50033 - Italienische Polizei führte Schlag gegen die ’Ndrangheta (2008-05-09)\n", + "Processing datasets/german-quotations/train/77549 - Gewalt auf den Straßen von Hongkong (2014-10-14).pretty.json\n", + "77549 - Gewalt auf den Straßen von Hongkong (2014-10-14)\n", + "Processing datasets/german-quotations/train/69231 - Terroranschläge in Norwegen: Polizei korrigiert Opferzahl auf 76 (2011-07-25).pretty.json\n", + "69231 - Terroranschläge in Norwegen: Polizei korrigiert Opferzahl auf 76 (2011-07-25)\n", + "Processing datasets/german-quotations/train/22943 - BND-Affäre: Am Montag tagt das parlamentarische Kontrollgremium (2006-03-01).pretty.json\n", + "22943 - BND-Affäre: Am Montag tagt das parlamentarische Kontrollgremium (2006-03-01)\n", + "Processing datasets/german-quotations/train/52153 - Prominente demonstrieren in Hamburg gegen das Tragen von Pelzen (2008-08-16).pretty.json\n", + "52153 - Prominente demonstrieren in Hamburg gegen das Tragen von Pelzen (2008-08-16)\n", + "Processing datasets/german-quotations/train/67437 - Plagiatsfall Guttenberg: Universität Bayreuth unterstellt „absichtliche Täuschung“ (2011-04-08).pretty.json\n", + "67437 - Plagiatsfall Guttenberg: Universität Bayreuth unterstellt „absichtliche Täuschung“ (2011-04-08)\n", + "Processing datasets/german-quotations/train/40973 - Debatte um Ex-RAF-Terroristin Susanne Albrecht (2007-05-07).pretty.json\n", + "40973 - Debatte um Ex-RAF-Terroristin Susanne Albrecht (2007-05-07)\n", + "Processing datasets/german-quotations/train/19569 - Erdbeben erschütterte Griechenland (2006-01-08).pretty.json\n", + "19569 - Erdbeben erschütterte Griechenland (2006-01-08)\n", + "Processing datasets/german-quotations/train/36491 - Fahren ohne Führerschein kann tödlich sein (2006-12-24).pretty.json\n", + "36491 - Fahren ohne Führerschein kann tödlich sein (2006-12-24)\n", + "Processing datasets/german-quotations/train/34728 - UNICEF stellte Studie vor: Gewalt bestimmt den Alltag vieler Kinder (2006-11-03).pretty.json\n", + "34728 - UNICEF stellte Studie vor: Gewalt bestimmt den Alltag vieler Kinder (2006-11-03)\n", + "Processing datasets/german-quotations/train/50935 - Brandenburgs Wirtschaftsminister Ulrich Junghanns bei Tesla Motors (2008-06-14).pretty.json\n", + "50935 - Brandenburgs Wirtschaftsminister Ulrich Junghanns bei Tesla Motors (2008-06-14)\n", + "Processing datasets/german-quotations/train/45420 - Neues Blu-ray-Disc-Profil 1.1 in Kraft getreten (2007-11-09).pretty.json\n", + "45420 - Neues Blu-ray-Disc-Profil 1.1 in Kraft getreten (2007-11-09)\n", + "Processing datasets/german-quotations/train/75701 - Ingolstadt: Ein Toter nach Busunglück auf der Autobahn 9 (2013-06-08).pretty.json\n", + "75701 - Ingolstadt: Ein Toter nach Busunglück auf der Autobahn 9 (2013-06-08)\n", + "Processing datasets/german-quotations/train/20929 - Spanisches Jagdflugzeug bei Übungsflug abgestürzt (2006-01-29).pretty.json\n", + "20929 - Spanisches Jagdflugzeug bei Übungsflug abgestürzt (2006-01-29)\n", + "Processing datasets/german-quotations/train/6196 - Michael Jackson „nicht schuldig“ gesprochen (2005-06-13).pretty.json\n", + "6196 - Michael Jackson „nicht schuldig“ gesprochen (2005-06-13)\n", + "Processing datasets/german-quotations/train/40269 - Vor 25 Jahren: Erstes deutsches Retortenbaby erblickt das Licht der Welt (2007-04-11).pretty.json\n", + "40269 - Vor 25 Jahren: Erstes deutsches Retortenbaby erblickt das Licht der Welt (2007-04-11)\n", + "Processing datasets/german-quotations/train/79859 - SPD-Bundestagsabgeordnete fälschte ihren Lebenslauf (2016-07-26).pretty.json\n", + "79859 - SPD-Bundestagsabgeordnete fälschte ihren Lebenslauf (2016-07-26)\n", + "Processing datasets/german-quotations/train/31906 - Flugzeug der Alaska Airlines nach Landung evakuiert (2006-08-20).pretty.json\n", + "31906 - Flugzeug der Alaska Airlines nach Landung evakuiert (2006-08-20)\n", + "Processing datasets/german-quotations/train/64070 - Hannelore Kraft neue Bundesratspräsidentin (2010-10-16).pretty.json\n", + "64070 - Hannelore Kraft neue Bundesratspräsidentin (2010-10-16)\n", + "Processing datasets/german-quotations/train/33668 - Der US-Amerikaner Edmund S. Phelps erhält den Nobelpreis für Wirtschaftswissenschaften (2006-10-11).pretty.json\n", + "33668 - Der US-Amerikaner Edmund S. Phelps erhält den Nobelpreis für Wirtschaftswissenschaften (2006-10-11)\n", + "Processing datasets/german-quotations/train/27811 - Wende der US-Diplomatie: USA zu direkten Gesprächen mit dem Iran bereit (2006-05-31).pretty.json\n", + "27811 - Wende der US-Diplomatie: USA zu direkten Gesprächen mit dem Iran bereit (2006-05-31)\n", + "Processing datasets/german-quotations/train/67143 - Tötungsdelikt in Krailling: Zwei Mädchen im Alter von acht und elf Jahren tot aufgefunden (2011-03-24).pretty.json\n", + "67143 - Tötungsdelikt in Krailling: Zwei Mädchen im Alter von acht und elf Jahren tot aufgefunden (2011-03-24)\n", + "Processing datasets/german-quotations/train/83753 - Handball-EM: Deutschland gewinnt Auftaktspiel gegen Neuling Niederlande (2020-01-11).pretty.json\n", + "83753 - Handball-EM: Deutschland gewinnt Auftaktspiel gegen Neuling Niederlande (2020-01-11)\n", + "Processing datasets/german-quotations/train/76792 - 10. Markt für Dresdner Geschichte und Geschichten: Der Verkehrsknoten Dresden und seine Geschichte (2014-03-23).pretty.json\n", + "76792 - 10. Markt für Dresdner Geschichte und Geschichten: Der Verkehrsknoten Dresden und seine Geschichte (2014-03-23)\n", + "Processing datasets/german-quotations/train/18551 - Grünen-Politiker wird neuer UN-Sondergesandter für Afghanistan (2005-12-29).pretty.json\n", + "18551 - Grünen-Politiker wird neuer UN-Sondergesandter für Afghanistan (2005-12-29)\n", + "Processing datasets/german-quotations/train/39074 - Wittlich: Rollerfahrer verunglückt (2007-03-16).pretty.json\n", + "39074 - Wittlich: Rollerfahrer verunglückt (2007-03-16)\n", + "Processing datasets/german-quotations/train/84585 - Gleichgeschlechtliche Ehe: Jetzt auch in Chile (2021-12-08).pretty.json\n", + "84585 - Gleichgeschlechtliche Ehe: Jetzt auch in Chile (2021-12-08)\n", + "Processing datasets/german-quotations/train/58874 - Schwulen-Demonstration in Belgrad aus Sicherheitsgründen abgesagt (2009-09-19).pretty.json\n", + "58874 - Schwulen-Demonstration in Belgrad aus Sicherheitsgründen abgesagt (2009-09-19)\n", + "Processing datasets/german-quotations/train/2668 - Bundespräsident Köhler warnt vor pauschaler Verurteilung deutscher Politiker (2005-01-10).pretty.json\n", + "2668 - Bundespräsident Köhler warnt vor pauschaler Verurteilung deutscher Politiker (2005-01-10)\n", + "Processing datasets/german-quotations/train/8659 - Irak: Zivilbevölkerung trägt Hauptlast des Terrors (2005-07-15).pretty.json\n", + "8659 - Irak: Zivilbevölkerung trägt Hauptlast des Terrors (2005-07-15)\n", + "Processing datasets/german-quotations/train/31283 - Britische Moslems warnen Premierminister Tony Blair wegen der Außenpolitik (2006-08-13).pretty.json\n", + "31283 - Britische Moslems warnen Premierminister Tony Blair wegen der Außenpolitik (2006-08-13)\n", + "Processing datasets/german-quotations/train/22629 - Cannabis: Expertentreffen in Bielefeld (2006-02-23).pretty.json\n", + "22629 - Cannabis: Expertentreffen in Bielefeld (2006-02-23)\n", + "Processing datasets/german-quotations/train/71677 - Bulgarien: Gesetz zur Konfiszierung von Mafiaeigentum verabschiedet (2012-02-17).pretty.json\n", + "71677 - Bulgarien: Gesetz zur Konfiszierung von Mafiaeigentum verabschiedet (2012-02-17)\n", + "Processing datasets/german-quotations/train/13194 - Vor dem Verfassungsreferendum im Irak – neue Welle von Anschlägen (2005-09-30).pretty.json\n", + "13194 - Vor dem Verfassungsreferendum im Irak – neue Welle von Anschlägen (2005-09-30)\n", + "Processing datasets/german-quotations/train/10535 - Neuer Weltrekord im Speerwerfen (2005-08-14).pretty.json\n", + "10535 - Neuer Weltrekord im Speerwerfen (2005-08-14)\n", + "Processing datasets/german-quotations/train/11775 - Bundesverwaltungsgericht kritisiert deutsche Unterstützung während des Irak-Krieges (2005-09-02).pretty.json\n", + "11775 - Bundesverwaltungsgericht kritisiert deutsche Unterstützung während des Irak-Krieges (2005-09-02)\n", + "Processing datasets/german-quotations/train/19094 - Cessna in indonesischer Provinz Papua abgestürzt (2006-01-05).pretty.json\n", + "19094 - Cessna in indonesischer Provinz Papua abgestürzt (2006-01-05)\n", + "Processing datasets/german-quotations/train/37082 - Panne bei Amazon.de (2007-01-12).pretty.json\n", + "37082 - Panne bei Amazon.de (2007-01-12)\n", + "Processing datasets/german-quotations/train/68437 - Motocross in Freising: ADAC MX Masters (2011-05-30).pretty.json\n", + "68437 - Motocross in Freising: ADAC MX Masters (2011-05-30)\n", + "Processing datasets/german-quotations/train/79030 - Fotostrecke: So bereitete sich Österreich auf den Eurovision Song Contest vor (2015-05-24).pretty.json\n", + "79030 - Fotostrecke: So bereitete sich Österreich auf den Eurovision Song Contest vor (2015-05-24)\n", + "Processing datasets/german-quotations/train/82126 - Saudi-Arabien führt Mehrwertsteuer ein (2018-01-06).pretty.json\n", + "82126 - Saudi-Arabien führt Mehrwertsteuer ein (2018-01-06)\n", + "Processing datasets/german-quotations/train/37353 - Kuipergürtel-Objekt soll Komet werden (2007-01-21).pretty.json\n", + "37353 - Kuipergürtel-Objekt soll Komet werden (2007-01-21)\n", + "Processing datasets/german-quotations/train/83739 - Al-Kuds-Führer Soleimani von US-Militär getötet (2020-01-05).pretty.json\n", + "83739 - Al-Kuds-Führer Soleimani von US-Militär getötet (2020-01-05)\n", + "Processing datasets/german-quotations/train/5203 - Uni-Rektorat-Besetzung in Freiburg dauert nun seit Montag an (2005-05-06).pretty.json\n", + "5203 - Uni-Rektorat-Besetzung in Freiburg dauert nun seit Montag an (2005-05-06)\n", + "Processing datasets/german-quotations/train/67057 - Radioaktives Jod im Trinkwasser der japanischen Hauptstadt (2011-03-19).pretty.json\n", + "67057 - Radioaktives Jod im Trinkwasser der japanischen Hauptstadt (2011-03-19)\n", + "Processing datasets/german-quotations/train/79793 - Glock gegen Werberat (2016-06-24).pretty.json\n", + "79793 - Glock gegen Werberat (2016-06-24)\n", + "Processing datasets/german-quotations/train/12814 - FDP-interner Machtkampf entschieden: Gerhardt weicht Westerwelle (2005-09-22).pretty.json\n", + "12814 - FDP-interner Machtkampf entschieden: Gerhardt weicht Westerwelle (2005-09-22)\n", + "Processing datasets/german-quotations/train/63989 - Gesine Lötzsch distanziert sich von Gregor Gysi (2010-10-14).pretty.json\n", + "63989 - Gesine Lötzsch distanziert sich von Gregor Gysi (2010-10-14)\n", + "Processing datasets/german-quotations/train/63418 - Unterstützung für “Die kubanischen Fünf“ (2010-08-19).pretty.json\n", + "63418 - Unterstützung für “Die kubanischen Fünf“ (2010-08-19)\n", + "Processing datasets/german-quotations/train/15420 - Gersfeld: Motorradfahrer tödlich verunglückt (2005-11-02).pretty.json\n", + "15420 - Gersfeld: Motorradfahrer tödlich verunglückt (2005-11-02)\n", + "Processing datasets/german-quotations/train/82511 - 105. Tour de France hat begonnen (2018-07-07).pretty.json\n", + "82511 - 105. Tour de France hat begonnen (2018-07-07)\n", + "Processing datasets/german-quotations/train/14742 - Aufsichtsrat der Bundesagentur für Arbeit fordert Lockerung des Datenschutzes (2005-10-25).pretty.json\n", + "14742 - Aufsichtsrat der Bundesagentur für Arbeit fordert Lockerung des Datenschutzes (2005-10-25)\n", + "Processing datasets/german-quotations/train/5419 - Deutsche Astronomen entdeckten Riesenplaneten (2005-05-18).pretty.json\n", + "5419 - Deutsche Astronomen entdeckten Riesenplaneten (2005-05-18)\n", + "Processing datasets/german-quotations/train/82520 - Dutzende Tote durch Unwetter in Japan (2018-07-09).pretty.json\n", + "82520 - Dutzende Tote durch Unwetter in Japan (2018-07-09)\n", + "Processing datasets/german-quotations/train/60304 - Airbus: Aus für A400M? (2010-01-05).pretty.json\n", + "60304 - Airbus: Aus für A400M? (2010-01-05)\n", + "Processing datasets/german-quotations/train/73273 - Justizopfer Horst Arnold ist tot (2012-07-05).pretty.json\n", + "73273 - Justizopfer Horst Arnold ist tot (2012-07-05)\n", + "Processing datasets/german-quotations/train/47908 - Berlin hat Klagefrist verschlafen – Reichsvermögen bleibt beim Bund (2008-02-09).pretty.json\n", + "47908 - Berlin hat Klagefrist verschlafen – Reichsvermögen bleibt beim Bund (2008-02-09)\n", + "Processing datasets/german-quotations/train/82138 - Diskussion um die Ausbeutung von Click- und Crowdworkern (2018-01-04).pretty.json\n", + "82138 - Diskussion um die Ausbeutung von Click- und Crowdworkern (2018-01-04)\n", + "Processing datasets/german-quotations/train/8183 - Brand im Londoner Hard Rock Café (2005-07-10).pretty.json\n", + "8183 - Brand im Londoner Hard Rock Café (2005-07-10)\n", + "Processing datasets/german-quotations/train/50736 - Spekulationen über Geheimtreffen zwischen Barack Obama und Hillary Clinton (2008-06-06).pretty.json\n", + "50736 - Spekulationen über Geheimtreffen zwischen Barack Obama und Hillary Clinton (2008-06-06)\n", + "Processing datasets/german-quotations/train/21016 - Roger Federer gewinnt die Australian Open (2006-01-29).pretty.json\n", + "21016 - Roger Federer gewinnt die Australian Open (2006-01-29)\n", + "Processing datasets/german-quotations/train/81851 - Ubuntu 17.10 „Artful Aardvark“ veröffentlicht (2017-11-05).pretty.json\n", + "81851 - Ubuntu 17.10 „Artful Aardvark“ veröffentlicht (2017-11-05)\n", + "Processing datasets/german-quotations/train/57933 - Massendemonstrationen gegen Wahlbetrug im Iran (2009-06-16).pretty.json\n", + "57933 - Massendemonstrationen gegen Wahlbetrug im Iran (2009-06-16)\n", + "Processing datasets/german-quotations/train/32117 - 2. Bundesliga: FC Carl Zeiss Jena schlägt 1. FC Köln mit 3:2 (2006-08-26).pretty.json\n", + "32117 - 2. Bundesliga: FC Carl Zeiss Jena schlägt 1. FC Köln mit 3:2 (2006-08-26)\n", + "Processing datasets/german-quotations/train/38073 - Koblenz: Ein Toter und sieben Schwerverletzte bei Wohnungsbrand (2007-02-11).pretty.json\n", + "38073 - Koblenz: Ein Toter und sieben Schwerverletzte bei Wohnungsbrand (2007-02-11)\n", + "Processing datasets/german-quotations/train/9093 - 61. Jahrestag des Hitler-Attentats (2005-07-22).pretty.json\n", + "9093 - 61. Jahrestag des Hitler-Attentats (2005-07-22)\n", + "Processing datasets/german-quotations/train/82125 - Betrunkener Autofahrer stößt Fußgängerin von Brücke (2018-01-02).pretty.json\n", + "82125 - Betrunkener Autofahrer stößt Fußgängerin von Brücke (2018-01-02)\n", + "Processing datasets/german-quotations/train/82556 - Großeinsatz für die Kaiserslauterer Feuerwehr bei sengender Hitze (2018-08-08).pretty.json\n", + "82556 - Großeinsatz für die Kaiserslauterer Feuerwehr bei sengender Hitze (2018-08-08)\n", + "Processing datasets/german-quotations/train/35750 - Verkehrsunfall in Kaiserslautern: Glück im Unglück (2006-12-02).pretty.json\n", + "35750 - Verkehrsunfall in Kaiserslautern: Glück im Unglück (2006-12-02)\n", + "Processing datasets/german-quotations/train/13440 - Union erneut im Streit um Gesundheitspolitik (2005-10-06).pretty.json\n", + "13440 - Union erneut im Streit um Gesundheitspolitik (2005-10-06)\n", + "Processing datasets/german-quotations/train/68713 - Generalstreik in Griechenland gegen Sparpaket (2011-06-16).pretty.json\n", + "68713 - Generalstreik in Griechenland gegen Sparpaket (2011-06-16)\n", + "Processing datasets/german-quotations/train/58800 - Windows 7 kommt am 22. Oktober in den Handel (2009-09-17).pretty.json\n", + "58800 - Windows 7 kommt am 22. Oktober in den Handel (2009-09-17)\n", + "Processing datasets/german-quotations/train/57026 - Bremen: Drei 18-Jährige von zwei Männern überfallen (2009-04-09).pretty.json\n", + "57026 - Bremen: Drei 18-Jährige von zwei Männern überfallen (2009-04-09)\n", + "Processing datasets/german-quotations/train/27869 - Toulouse: Explosion im Airbus-Werk (2006-06-02).pretty.json\n", + "27869 - Toulouse: Explosion im Airbus-Werk (2006-06-02)\n", + "Processing datasets/german-quotations/train/12120 - Apple bringt „iPod nano“ auf den Markt (2005-09-11).pretty.json\n", + "12120 - Apple bringt „iPod nano“ auf den Markt (2005-09-11)\n", + "Processing datasets/german-quotations/train/23333 - Verteidigungsminister der EU noch uneinig über eine Kongomission (2006-03-07).pretty.json\n", + "23333 - Verteidigungsminister der EU noch uneinig über eine Kongomission (2006-03-07)\n", + "Processing datasets/german-quotations/train/25575 - Rauchen auf dem Flughafen Brüssel seit Karsamstag verboten (2006-04-19).pretty.json\n", + "25575 - Rauchen auf dem Flughafen Brüssel seit Karsamstag verboten (2006-04-19)\n", + "Processing datasets/german-quotations/train/45244 - Togliatti (Russland): Bombenanschlag auf Bus (2007-10-31).pretty.json\n", + "45244 - Togliatti (Russland): Bombenanschlag auf Bus (2007-10-31)\n", + "Processing datasets/german-quotations/train/7546 - Heikle Situation vor der Vertrauensfrage des Kanzlers (2005-06-30).pretty.json\n", + "7546 - Heikle Situation vor der Vertrauensfrage des Kanzlers (2005-06-30)\n", + "Processing datasets/german-quotations/train/23714 - Nächste Spaceshuttle-Mission STS-121 erneut verschoben (2006-03-15).pretty.json\n", + "23714 - Nächste Spaceshuttle-Mission STS-121 erneut verschoben (2006-03-15)\n", + "Processing datasets/german-quotations/train/80391 - Irans ehemaliger Präsident Rafsandschani ist tot (2017-01-08).pretty.json\n", + "80391 - Irans ehemaliger Präsident Rafsandschani ist tot (2017-01-08)\n", + "Processing datasets/german-quotations/train/55265 - Batteriehersteller gibt Finanzspritze für Elektroautobauer Think Global (2009-01-16).pretty.json\n", + "55265 - Batteriehersteller gibt Finanzspritze für Elektroautobauer Think Global (2009-01-16)\n", + "Processing datasets/german-quotations/train/73475 - EU zahlt trotz Krise weiter Subventionen an Griechenland (2012-07-24).pretty.json\n", + "73475 - EU zahlt trotz Krise weiter Subventionen an Griechenland (2012-07-24)\n", + "Processing datasets/german-quotations/train/71365 - Unterfranken: Rotorblatt eines 80 Meter hohen Windkraftrades abgebrochen (2012-01-24).pretty.json\n", + "71365 - Unterfranken: Rotorblatt eines 80 Meter hohen Windkraftrades abgebrochen (2012-01-24)\n", + "Processing datasets/german-quotations/train/14955 - Vogelgrippe: Entwarnung bei Chinesin und Franzosen (2005-10-27).pretty.json\n", + "14955 - Vogelgrippe: Entwarnung bei Chinesin und Franzosen (2005-10-27)\n", + "Processing datasets/german-quotations/train/81501 - Der Absatz von Borgward brach im Juli ein (2017-08-19).pretty.json\n", + "81501 - Der Absatz von Borgward brach im Juli ein (2017-08-19)\n", + "Processing datasets/german-quotations/train/15980 - Erneutes Erdbeben in der Rheinebene (2005-11-12).pretty.json\n", + "15980 - Erneutes Erdbeben in der Rheinebene (2005-11-12)\n", + "Processing datasets/german-quotations/train/5583 - Indianer haben wenige Urahnen (2005-05-25).pretty.json\n", + "5583 - Indianer haben wenige Urahnen (2005-05-25)\n", + "Processing datasets/german-quotations/train/50226 - Weiterhin katastrophale Lage in Myanmar nach dem Zyklon „Nargis“ (2008-05-16).pretty.json\n", + "50226 - Weiterhin katastrophale Lage in Myanmar nach dem Zyklon „Nargis“ (2008-05-16)\n", + "Processing datasets/german-quotations/train/79990 - Wackelt der Stuhl von Justizminister Heiko Maas? (2016-09-08).pretty.json\n", + "79990 - Wackelt der Stuhl von Justizminister Heiko Maas? (2016-09-08)\n", + "Processing datasets/german-quotations/train/12554 - Flugverbot für Cameroon Airlines in Frankreich (2005-09-17).pretty.json\n", + "12554 - Flugverbot für Cameroon Airlines in Frankreich (2005-09-17)\n", + "Processing datasets/german-quotations/train/5049 - FC Bayern München vorzeitig deutscher Fußball-Meister (2005-05-01).pretty.json\n", + "5049 - FC Bayern München vorzeitig deutscher Fußball-Meister (2005-05-01)\n", + "Processing datasets/german-quotations/train/13682 - Serieneinbrüche im Raum Knittelfeld (2005-10-10).pretty.json\n", + "13682 - Serieneinbrüche im Raum Knittelfeld (2005-10-10)\n", + "Processing datasets/german-quotations/train/32592 - Haushaltsdebatte 2007: „Ja, es gibt Zumutungen“ (2006-09-05).pretty.json\n", + "32592 - Haushaltsdebatte 2007: „Ja, es gibt Zumutungen“ (2006-09-05)\n", + "Processing datasets/german-quotations/train/75225 - Das Weltsozialforum in Tunesien will den Arabischen Frühling neu beleben (2013-03-27).pretty.json\n", + "75225 - Das Weltsozialforum in Tunesien will den Arabischen Frühling neu beleben (2013-03-27)\n", + "Processing datasets/german-quotations/train/68040 - Hohe Waldbrandgefahr in weiten Teilen Deutschlands (2011-05-09).pretty.json\n", + "68040 - Hohe Waldbrandgefahr in weiten Teilen Deutschlands (2011-05-09)\n", + "Processing datasets/german-quotations/train/28736 - Europäisch-afrikanische Migrationskonferenz beginnt im Juli (2006-06-21).pretty.json\n", + "28736 - Europäisch-afrikanische Migrationskonferenz beginnt im Juli (2006-06-21)\n", + "Processing datasets/german-quotations/train/67724 - Ungarn: Rechtsextreme Gruppe errichtet Trainingscamp (2011-04-23).pretty.json\n", + "67724 - Ungarn: Rechtsextreme Gruppe errichtet Trainingscamp (2011-04-23)\n", + "Processing datasets/german-quotations/train/41694 - Demonstration gegen das ASEM-Treffen in Hamburg (2007-05-28).pretty.json\n", + "41694 - Demonstration gegen das ASEM-Treffen in Hamburg (2007-05-28)\n", + "Processing datasets/german-quotations/train/6439 - EU-Gipfel scheitert an fehlender Kompromissbereitschaft (2005-06-18).pretty.json\n", + "6439 - EU-Gipfel scheitert an fehlender Kompromissbereitschaft (2005-06-18)\n", + "Processing datasets/german-quotations/train/25118 - Zehn Soldaten bei Hubschrauberabsturz in Nepal getötet (2006-04-08).pretty.json\n", + "25118 - Zehn Soldaten bei Hubschrauberabsturz in Nepal getötet (2006-04-08)\n", + "Processing datasets/german-quotations/train/21224 - Experte: Falscher Leim ist für den Einsturz der Eissporthalle verantwortlich (2006-02-01).pretty.json\n", + "21224 - Experte: Falscher Leim ist für den Einsturz der Eissporthalle verantwortlich (2006-02-01)\n", + "Processing datasets/german-quotations/train/50049 - Kaiserslautern: Benzin als Grillanzünder führt zu erheblichem Brandschaden (2008-05-11).pretty.json\n", + "50049 - Kaiserslautern: Benzin als Grillanzünder führt zu erheblichem Brandschaden (2008-05-11)\n", + "Processing datasets/german-quotations/train/35901 - In Deutschland sind 13 Prozent der Bevölkerung armutsgefährdet (2006-12-05).pretty.json\n", + "35901 - In Deutschland sind 13 Prozent der Bevölkerung armutsgefährdet (2006-12-05)\n", + "Processing datasets/german-quotations/train/33980 - Singapur in Rauchschwaden eingehüllt (2006-10-17).pretty.json\n", + "33980 - Singapur in Rauchschwaden eingehüllt (2006-10-17)\n", + "Processing datasets/german-quotations/train/48348 - Spanische Parlamentswahlen rücken näher (2008-02-27).pretty.json\n", + "48348 - Spanische Parlamentswahlen rücken näher (2008-02-27)\n", + "Processing datasets/german-quotations/train/35566 - S-Bahn-Gleisbauarbeiten in Bad Homburg abgeschlossen (2006-11-29).pretty.json\n", + "35566 - S-Bahn-Gleisbauarbeiten in Bad Homburg abgeschlossen (2006-11-29)\n", + "Processing datasets/german-quotations/train/61889 - US-Polizei verhaftet Mitglieder einer christlichen paramilitärischen Gruppe (2010-03-30).pretty.json\n", + "61889 - US-Polizei verhaftet Mitglieder einer christlichen paramilitärischen Gruppe (2010-03-30)\n", + "Processing datasets/german-quotations/train/44338 - Blauzungenkrankheit jetzt auch in Baden-Württemberg nachgewiesen (2007-09-19).pretty.json\n", + "44338 - Blauzungenkrankheit jetzt auch in Baden-Württemberg nachgewiesen (2007-09-19)\n", + "Processing datasets/german-quotations/train/12291 - Grüne wollen Tankstellen zu Biodiesel-Verkauf zwingen (2005-09-10).pretty.json\n", + "12291 - Grüne wollen Tankstellen zu Biodiesel-Verkauf zwingen (2005-09-10)\n", + "Processing datasets/german-quotations/train/26226 - Schweizer Politiker Joseph Deiss kündigte Rücktritt an (2006-05-04).pretty.json\n", + "26226 - Schweizer Politiker Joseph Deiss kündigte Rücktritt an (2006-05-04)\n", + "Processing datasets/german-quotations/train/59410 - Swantje Hartmann vom Vorstand der niedersächsischen SPD-Landtagsfraktion zurückgetreten (2008-07-08).pretty.json\n", + "59410 - Swantje Hartmann vom Vorstand der niedersächsischen SPD-Landtagsfraktion zurückgetreten (2008-07-08)\n", + "Processing datasets/german-quotations/train/14127 - Ferrari-Teamchef stellt sich vor Michael Schumacher (2005-10-18).pretty.json\n", + "14127 - Ferrari-Teamchef stellt sich vor Michael Schumacher (2005-10-18)\n", + "Processing datasets/german-quotations/train/19599 - US-Hubschrauber mit zwölf Menschen an Bord im Irak abgestürzt (2006-01-09).pretty.json\n", + "19599 - US-Hubschrauber mit zwölf Menschen an Bord im Irak abgestürzt (2006-01-09)\n", + "Processing datasets/german-quotations/train/55730 - Kenia: Über hundert Menschen sterben nach Tankwagenunglück (2009-02-01).pretty.json\n", + "55730 - Kenia: Über hundert Menschen sterben nach Tankwagenunglück (2009-02-01)\n", + "Processing datasets/german-quotations/train/81706 - Immer mehr Stars lassen sich Tätowieren – Schmuck, Modeerscheinung oder Tieferer Sinn? (2017-10-12).pretty.json\n", + "81706 - Immer mehr Stars lassen sich Tätowieren – Schmuck, Modeerscheinung oder Tieferer Sinn? (2017-10-12)\n", + "Processing datasets/german-quotations/train/64137 - Studie: Steinzeitmenschen aßen nicht nur Fleisch (2010-10-19).pretty.json\n", + "64137 - Studie: Steinzeitmenschen aßen nicht nur Fleisch (2010-10-19)\n", + "Processing datasets/german-quotations/train/5114 - Papst-Golf bei eBay auf 60.000 Euro gestiegen (2005-05-02).pretty.json\n", + "5114 - Papst-Golf bei eBay auf 60.000 Euro gestiegen (2005-05-02)\n", + "Processing datasets/german-quotations/train/34828 - Fresenius AG plant Umwandlung der Rechtsform und Neuordnung des Stammkapitals (2006-11-10).pretty.json\n", + "34828 - Fresenius AG plant Umwandlung der Rechtsform und Neuordnung des Stammkapitals (2006-11-10)\n", + "Processing datasets/german-quotations/train/49378 - Nach Milliardenverlusten bei der Schweizer Bank UBS: Bankchef Marcel Ospel zurückgetreten (2008-04-01).pretty.json\n", + "49378 - Nach Milliardenverlusten bei der Schweizer Bank UBS: Bankchef Marcel Ospel zurückgetreten (2008-04-01)\n", + "Processing datasets/german-quotations/train/54204 - Schwerer Verkehrsunfall auf der L 502 in Kaiserslautern (2008-11-23).pretty.json\n", + "54204 - Schwerer Verkehrsunfall auf der L 502 in Kaiserslautern (2008-11-23)\n", + "Processing datasets/german-quotations/train/43411 - 75 Prozent der Abgeordneten für Offenlegung ihrer Nebeneinkünfte (2007-08-08).pretty.json\n", + "43411 - 75 Prozent der Abgeordneten für Offenlegung ihrer Nebeneinkünfte (2007-08-08)\n", + "Processing datasets/german-quotations/train/59662 - USA: Der demokratische Senator Christopher Dodd will die „Fed“ entmachten (2009-11-13).pretty.json\n", + "59662 - USA: Der demokratische Senator Christopher Dodd will die „Fed“ entmachten (2009-11-13)\n", + "Processing datasets/german-quotations/train/49204 - In Rio de Janeiro grassiert das Dengue-Fieber (2008-03-25).pretty.json\n", + "49204 - In Rio de Janeiro grassiert das Dengue-Fieber (2008-03-25)\n", + "Processing datasets/german-quotations/train/12667 - ARD-Hochrechnungen doch mit Microsoft-Logo ausgestrahlt (2005-09-19).pretty.json\n", + "12667 - ARD-Hochrechnungen doch mit Microsoft-Logo ausgestrahlt (2005-09-19)\n", + "Processing datasets/german-quotations/train/49854 - Erneute Schulschließung wegen Masern an einer Freiburger Schule (2008-04-30).pretty.json\n", + "49854 - Erneute Schulschließung wegen Masern an einer Freiburger Schule (2008-04-30)\n", + "Processing datasets/german-quotations/train/58768 - Polizeigewalt am Rande der Demonstration „Freiheit statt Angst“ in Berlin (2009-09-14).pretty.json\n", + "58768 - Polizeigewalt am Rande der Demonstration „Freiheit statt Angst“ in Berlin (2009-09-14)\n", + "Processing datasets/german-quotations/train/35600 - Die Volkswagen AG baut in Indien eine neue Produktionsstätte (2006-11-30).pretty.json\n", + "35600 - Die Volkswagen AG baut in Indien eine neue Produktionsstätte (2006-11-30)\n", + "Processing datasets/german-quotations/train/3002 - Staatsstreich in Togo (2005-02-06).pretty.json\n", + "3002 - Staatsstreich in Togo (2005-02-06)\n", + "Processing datasets/german-quotations/train/13761 - Streit um Richtlinienkompetenz einer Bundeskanzlerin Angela Merkel (2005-10-12).pretty.json\n", + "13761 - Streit um Richtlinienkompetenz einer Bundeskanzlerin Angela Merkel (2005-10-12)\n", + "Processing datasets/german-quotations/train/50858 - Tesla Motors startet PR-Kampagne in Europa (2008-06-12).pretty.json\n", + "50858 - Tesla Motors startet PR-Kampagne in Europa (2008-06-12)\n", + "Processing datasets/german-quotations/train/68811 - Stadtrat Dresden beschließt Entwicklung eines CO2-neutralen Stadtteils (2011-06-24).pretty.json\n", + "68811 - Stadtrat Dresden beschließt Entwicklung eines CO2-neutralen Stadtteils (2011-06-24)\n", + "Processing datasets/german-quotations/train/25158 - Razzia bei EADS und Airbus (2006-04-11).pretty.json\n", + "25158 - Razzia bei EADS und Airbus (2006-04-11)\n", + "Processing datasets/german-quotations/train/72118 - Mitt Romney gewinnt Vorwahl in Illinois (2012-03-23).pretty.json\n", + "72118 - Mitt Romney gewinnt Vorwahl in Illinois (2012-03-23)\n", + "Processing datasets/german-quotations/train/6659 - Afrikaner wird neuer Chef der UNIDO (2005-06-22).pretty.json\n", + "6659 - Afrikaner wird neuer Chef der UNIDO (2005-06-22)\n", + "Processing datasets/german-quotations/train/12325 - Hurrikan Katrina: USA lehnten Lebensmittelhilfe aus Deutschland ab (2005-09-10).pretty.json\n", + "12325 - Hurrikan Katrina: USA lehnten Lebensmittelhilfe aus Deutschland ab (2005-09-10)\n", + "Processing datasets/german-quotations/train/46316 - Saudi-Arabien: König begnadigt „Qatif Girl“ (2007-12-17).pretty.json\n", + "46316 - Saudi-Arabien: König begnadigt „Qatif Girl“ (2007-12-17)\n", + "Processing datasets/german-quotations/train/59896 - Schwerverbrecher Michalski und Heckhoff wieder gefasst (2009-12-02).pretty.json\n", + "59896 - Schwerverbrecher Michalski und Heckhoff wieder gefasst (2009-12-02)\n", + "Processing datasets/german-quotations/train/80813 - Zunehmende Angriffe gegen Juden in Deutschland (2017-04-04).pretty.json\n", + "80813 - Zunehmende Angriffe gegen Juden in Deutschland (2017-04-04)\n", + "Processing datasets/german-quotations/train/9907 - Passagierflugzeug in Leeds notgelandet (2005-08-04).pretty.json\n", + "9907 - Passagierflugzeug in Leeds notgelandet (2005-08-04)\n", + "Processing datasets/german-quotations/train/33932 - Großrazzia in Wieslocher Discothek (2006-10-17).pretty.json\n", + "33932 - Großrazzia in Wieslocher Discothek (2006-10-17)\n", + "Processing datasets/german-quotations/train/8845 - Indiens Trägerraketenprogramm vor weiteren Tests (2005-07-18).pretty.json\n", + "8845 - Indiens Trägerraketenprogramm vor weiteren Tests (2005-07-18)\n", + "Processing datasets/german-quotations/train/30456 - Möglicherweise Wrack des Flugzeugträgers „Graf Zeppelin“ entdeckt (2006-07-26).pretty.json\n", + "30456 - Möglicherweise Wrack des Flugzeugträgers „Graf Zeppelin“ entdeckt (2006-07-26)\n", + "Processing datasets/german-quotations/train/24326 - Bangkok: Demonstranten fordern vom König die Einsetzung einer Interimsregierung (2006-03-26).pretty.json\n", + "24326 - Bangkok: Demonstranten fordern vom König die Einsetzung einer Interimsregierung (2006-03-26)\n", + "Processing datasets/german-quotations/train/33446 - Ayaan Hirsi Ali erhielt Preis der Stadt Kassel (2006-10-02).pretty.json\n", + "33446 - Ayaan Hirsi Ali erhielt Preis der Stadt Kassel (2006-10-02)\n", + "Processing datasets/german-quotations/train/80622 - Ex-Verkehrsminister Günther Krause wurde Opfer eines Betrügers (2017-03-07).pretty.json\n", + "80622 - Ex-Verkehrsminister Günther Krause wurde Opfer eines Betrügers (2017-03-07)\n", + "Processing datasets/german-quotations/train/12873 - Notlandung auf dem Hamburger Flughafen (2005-09-24).pretty.json\n", + "12873 - Notlandung auf dem Hamburger Flughafen (2005-09-24)\n", + "Processing datasets/german-quotations/train/27237 - Braunbär in Deutschland: Freigabe zum Abschuss (2006-05-22).pretty.json\n", + "27237 - Braunbär in Deutschland: Freigabe zum Abschuss (2006-05-22)\n", + "Processing datasets/german-quotations/train/27258 - Danziger Katharinenkirche bei Brand schwer beschädigt (2006-05-22).pretty.json\n", + "27258 - Danziger Katharinenkirche bei Brand schwer beschädigt (2006-05-22)\n", + "Processing datasets/german-quotations/train/67014 - UN-Sicherheitsrat beschließt Einrichtung einer Flugverbotszone über Libyen (2011-03-18).pretty.json\n", + "67014 - UN-Sicherheitsrat beschließt Einrichtung einer Flugverbotszone über Libyen (2011-03-18)\n", + "Processing datasets/german-quotations/train/42492 - Kernkraftwerk Brunsbüttel geht nach Störung wieder ans Netz (2007-07-01).pretty.json\n", + "42492 - Kernkraftwerk Brunsbüttel geht nach Störung wieder ans Netz (2007-07-01)\n", + "Processing datasets/german-quotations/train/74982 - NASA beobachtet ISON-Kometen im Weltraum (2013-02-25).pretty.json\n", + "74982 - NASA beobachtet ISON-Kometen im Weltraum (2013-02-25)\n", + "Processing datasets/german-quotations/train/20735 - Zwei deutsche Ingenieure im Irak entführt (2006-01-24).pretty.json\n", + "20735 - Zwei deutsche Ingenieure im Irak entführt (2006-01-24)\n", + "Processing datasets/german-quotations/train/47946 - London: Großbrand auf Straßenmarkt in Camden (2008-02-11).pretty.json\n", + "47946 - London: Großbrand auf Straßenmarkt in Camden (2008-02-11)\n", + "Processing datasets/german-quotations/train/30404 - Zustand Ariel Scharons deutlich verschlechtert (2006-07-24).pretty.json\n", + "30404 - Zustand Ariel Scharons deutlich verschlechtert (2006-07-24)\n", + "Processing datasets/german-quotations/train/51314 - Frieda Borchert ist mit 111 Jahren in Berlin verstorben (2008-06-26).pretty.json\n", + "51314 - Frieda Borchert ist mit 111 Jahren in Berlin verstorben (2008-06-26)\n", + "Processing datasets/german-quotations/train/63500 - Bayerischer Grünen-Politiker Sepp Daxenberger gestorben (2010-08-19).pretty.json\n", + "63500 - Bayerischer Grünen-Politiker Sepp Daxenberger gestorben (2010-08-19)\n", + "Processing datasets/german-quotations/train/43134 - Thailand: Thai-Rak-Thai-Mitglieder suchen neue organisatorische Basis (2007-07-29).pretty.json\n", + "43134 - Thailand: Thai-Rak-Thai-Mitglieder suchen neue organisatorische Basis (2007-07-29)\n", + "Processing datasets/german-quotations/train/18890 - Bad Reichenhall: Bergungsarbeiten in der Eissporthalle dauern an (2006-01-03).pretty.json\n", + "18890 - Bad Reichenhall: Bergungsarbeiten in der Eissporthalle dauern an (2006-01-03)\n", + "Processing datasets/german-quotations/train/6062 - Einigung über Bundestagswahlbündnis PDS-WASG bis Samstag möglich (2005-06-09).pretty.json\n", + "6062 - Einigung über Bundestagswahlbündnis PDS-WASG bis Samstag möglich (2005-06-09)\n", + "Processing datasets/german-quotations/train/70653 - Deutscher Aktienindex im Plus (2011-12-02).pretty.json\n", + "70653 - Deutscher Aktienindex im Plus (2011-12-02)\n", + "Processing datasets/german-quotations/train/14990 - Österreichische Forscher entwickeln Plasmaantrieb (2005-10-28).pretty.json\n", + "14990 - Österreichische Forscher entwickeln Plasmaantrieb (2005-10-28)\n", + "Processing datasets/german-quotations/train/9006 - EBay verzeichnet einen Gewinnsprung von 53 Prozent (2005-07-21).pretty.json\n", + "9006 - EBay verzeichnet einen Gewinnsprung von 53 Prozent (2005-07-21)\n", + "Processing datasets/german-quotations/train/69856 - Bulgarien: Berüchtigter Roma-Boss festgenommen - Proteste gegen die Minderheit überschatten Wahlkampf (2011-09-28).pretty.json\n", + "69856 - Bulgarien: Berüchtigter Roma-Boss festgenommen - Proteste gegen die Minderheit überschatten Wahlkampf (2011-09-28)\n", + "Processing datasets/german-quotations/train/77153 - Kommt es in Sachsen zur schwarz-grünen Koalition? (2014-08-15).pretty.json\n", + "77153 - Kommt es in Sachsen zur schwarz-grünen Koalition? (2014-08-15)\n", + "Processing datasets/german-quotations/train/61550 - Terrorismus in Deutschland: Langjährige Freiheitsstrafen gegen Mitglieder der so genannten Sauerland-Gruppe (2010-03-04).pretty.json\n", + "61550 - Terrorismus in Deutschland: Langjährige Freiheitsstrafen gegen Mitglieder der so genannten Sauerland-Gruppe (2010-03-04)\n", + "Processing datasets/german-quotations/train/2124 - Handball: Wegen Lizenzverstoßes acht Punkte Abzug für den HSV (2004-12-23).pretty.json\n", + "2124 - Handball: Wegen Lizenzverstoßes acht Punkte Abzug für den HSV (2004-12-23)\n", + "Processing datasets/german-quotations/train/69725 - Mann soll Tochter 34 Jahre lang vergewaltigt haben (2011-09-14).pretty.json\n", + "69725 - Mann soll Tochter 34 Jahre lang vergewaltigt haben (2011-09-14)\n", + "Processing datasets/german-quotations/train/77468 - Fotostrecke: Proteste in Hong Kong am 27. und 28. September 2014 (2014-09-29).pretty.json\n", + "77468 - Fotostrecke: Proteste in Hong Kong am 27. und 28. September 2014 (2014-09-29)\n", + "Processing datasets/german-quotations/train/52016 - Olympische Spiele: Thailänderin holt erstes Gold für ihr Land im Gewichtheben (2008-08-10).pretty.json\n", + "52016 - Olympische Spiele: Thailänderin holt erstes Gold für ihr Land im Gewichtheben (2008-08-10)\n", + "Processing datasets/german-quotations/train/61557 - BASF hofft auf weitere Genehmigungen für gentechnisch veränderte Kartoffeln (2010-03-04).pretty.json\n", + "61557 - BASF hofft auf weitere Genehmigungen für gentechnisch veränderte Kartoffeln (2010-03-04)\n", + "Processing datasets/german-quotations/train/25991 - Der FC Bayern München gewinnt den DFB-Pokal 2006 (2006-05-01).pretty.json\n", + "25991 - Der FC Bayern München gewinnt den DFB-Pokal 2006 (2006-05-01)\n", + "Processing datasets/german-quotations/train/33563 - Notlandung einer Boeing 757 in Budapest (2006-10-05).pretty.json\n", + "33563 - Notlandung einer Boeing 757 in Budapest (2006-10-05)\n", + "Processing datasets/german-quotations/train/43684 - Pétanque: EuroCup-Endrunde findet in Rastatt statt (2007-08-20).pretty.json\n", + "43684 - Pétanque: EuroCup-Endrunde findet in Rastatt statt (2007-08-20)\n", + "Processing datasets/german-quotations/train/9251 - Thailands Ministerpräsident unter Druck wegen Notstandsvollmachten (2005-07-24).pretty.json\n", + "9251 - Thailands Ministerpräsident unter Druck wegen Notstandsvollmachten (2005-07-24)\n", + "Processing datasets/german-quotations/train/49334 - Interview mit dem deutschen Regisseur von Musikvideos, Uwe Flade (2008-03-30).pretty.json\n", + "49334 - Interview mit dem deutschen Regisseur von Musikvideos, Uwe Flade (2008-03-30)\n", + "Processing datasets/german-quotations/train/38651 - Internationaler Gerichtshof: Massaker von Srebrenica im Jahre 1995 war Völkermord (2007-03-02).pretty.json\n", + "38651 - Internationaler Gerichtshof: Massaker von Srebrenica im Jahre 1995 war Völkermord (2007-03-02)\n", + "Processing datasets/german-quotations/train/45626 - Franz Müntefering tritt von seinen Ämtern zurück (2007-11-13).pretty.json\n", + "45626 - Franz Müntefering tritt von seinen Ämtern zurück (2007-11-13)\n", + "Processing datasets/german-quotations/train/73463 - München: Zehnte Stadtmeisterschaft im Klettern (2012-07-22).pretty.json\n", + "73463 - München: Zehnte Stadtmeisterschaft im Klettern (2012-07-22)\n", + "Processing datasets/german-quotations/train/45609 - Großes Feuer brennt in London (2007-11-12).pretty.json\n", + "45609 - Großes Feuer brennt in London (2007-11-12)\n", + "Processing datasets/german-quotations/train/35318 - Klaus Wowereit fiel bei Wiederwahl im ersten Wahlgang durch – im zweiten erhielt er genügend Stimmen (2006-11-23).pretty.json\n", + "35318 - Klaus Wowereit fiel bei Wiederwahl im ersten Wahlgang durch – im zweiten erhielt er genügend Stimmen (2006-11-23)\n", + "Processing datasets/german-quotations/train/80937 - Tesla: Droht nach Übernahme ein Streik in Deutschland? (2017-04-17).pretty.json\n", + "80937 - Tesla: Droht nach Übernahme ein Streik in Deutschland? (2017-04-17)\n", + "Processing datasets/german-quotations/train/64617 - Zwölfter Spieltag der Fußball-Bundesliga in der Saison 2010-11 (2010-11-18).pretty.json\n", + "64617 - Zwölfter Spieltag der Fußball-Bundesliga in der Saison 2010-11 (2010-11-18)\n", + "Processing datasets/german-quotations/train/38621 - Dengue-Fieber: Paraguay ruft den Notstand aus (2007-02-27).pretty.json\n", + "38621 - Dengue-Fieber: Paraguay ruft den Notstand aus (2007-02-27)\n", + "Processing datasets/german-quotations/train/23657 - Aktionen gegen Verzögerung der Veröffentlichungspflicht von Abgeordnetengehältern (2005-03-13).pretty.json\n", + "23657 - Aktionen gegen Verzögerung der Veröffentlichungspflicht von Abgeordnetengehältern (2005-03-13)\n", + "Processing datasets/german-quotations/train/21661 - Torwart des SV Werder Bremen schwer verletzt in Klinik eingeliefert (2006-02-09).pretty.json\n", + "21661 - Torwart des SV Werder Bremen schwer verletzt in Klinik eingeliefert (2006-02-09)\n", + "Processing datasets/german-quotations/train/80548 - Chinesischer Bauer verklagt Chemiegiganten (2017-02-24).pretty.json\n", + "80548 - Chinesischer Bauer verklagt Chemiegiganten (2017-02-24)\n", + "Processing datasets/german-quotations/train/30253 - Der „Southwest Regional Spaceport“ heißt jetzt „Spaceport America“ (2006-07-21).pretty.json\n", + "30253 - Der „Southwest Regional Spaceport“ heißt jetzt „Spaceport America“ (2006-07-21)\n", + "Processing datasets/german-quotations/train/70416 - Hochbrisante Datei: Planten die Neonazis aus Thüringen Anschläge auf Politiker? (2011-11-16).pretty.json\n", + "70416 - Hochbrisante Datei: Planten die Neonazis aus Thüringen Anschläge auf Politiker? (2011-11-16)\n", + "Processing datasets/german-quotations/train/38926 - Deutscher in Thailand wegen Kindesmissbrauchs festgenommen (2007-03-09).pretty.json\n", + "38926 - Deutscher in Thailand wegen Kindesmissbrauchs festgenommen (2007-03-09)\n", + "Processing datasets/german-quotations/train/83502 - Italien: Neuwahlen vorerst abgewendet (2019-09-02).pretty.json\n", + "83502 - Italien: Neuwahlen vorerst abgewendet (2019-09-02)\n", + "Processing datasets/german-quotations/train/46722 - Erde am 3. Januar in Sonnennähe (2008-01-04).pretty.json\n", + "46722 - Erde am 3. Januar in Sonnennähe (2008-01-04)\n", + "Processing datasets/german-quotations/train/6662 - Österreich: Ursache für Explosion in Stahlwerk ermittelt (2005-04-09).pretty.json\n", + "6662 - Österreich: Ursache für Explosion in Stahlwerk ermittelt (2005-04-09)\n", + "Processing datasets/german-quotations/train/71448 - Europa: Kältewelle fordert Todesopfer (2012-01-31).pretty.json\n", + "71448 - Europa: Kältewelle fordert Todesopfer (2012-01-31)\n", + "Processing datasets/german-quotations/train/33338 - UP Aerospace veröffentlicht erste Analyse des gescheiterten Erstfluges (2006-10-01).pretty.json\n", + "33338 - UP Aerospace veröffentlicht erste Analyse des gescheiterten Erstfluges (2006-10-01)\n", + "Processing datasets/german-quotations/train/2937 - Nürnberger Bürger müssen tiefer in die Tasche greifen (2005-02-02).pretty.json\n", + "2937 - Nürnberger Bürger müssen tiefer in die Tasche greifen (2005-02-02)\n", + "Processing datasets/german-quotations/train/31223 - Bank von England veröffentlichte Namensliste von mutmaßlichen Terroristen (2006-08-13).pretty.json\n", + "31223 - Bank von England veröffentlichte Namensliste von mutmaßlichen Terroristen (2006-08-13)\n", + "Processing datasets/german-quotations/train/28493 - Wien: Komponist György Ligeti gestorben (2006-06-15).pretty.json\n", + "28493 - Wien: Komponist György Ligeti gestorben (2006-06-15)\n", + "Processing datasets/german-quotations/train/33276 - Ermittlungsergebnis wegen Hubschrauberabsturz bei Weilheim-Teck (2006-09-27).pretty.json\n", + "33276 - Ermittlungsergebnis wegen Hubschrauberabsturz bei Weilheim-Teck (2006-09-27)\n", + "Processing datasets/german-quotations/train/37132 - Aktionen gegen Rechts in Magdeburg (2007-01-13).pretty.json\n", + "37132 - Aktionen gegen Rechts in Magdeburg (2007-01-13)\n", + "Processing datasets/german-quotations/train/48551 - Der deutsche Sänger Ivan Rebroff ist tot (2008-02-29).pretty.json\n", + "48551 - Der deutsche Sänger Ivan Rebroff ist tot (2008-02-29)\n", + "Processing datasets/german-quotations/train/12980 - Kleinflugzeug stürzte in Nordschweden ab (2005-09-26).pretty.json\n", + "12980 - Kleinflugzeug stürzte in Nordschweden ab (2005-09-26)\n", + "Processing datasets/german-quotations/train/47835 - Baden-Württemberg: Schüler sollen keine Hausaufgaben mehr bekommen (2008-02-07).pretty.json\n", + "47835 - Baden-Württemberg: Schüler sollen keine Hausaufgaben mehr bekommen (2008-02-07)\n", + "Processing datasets/german-quotations/train/51446 - Studie: „Große grundsätzliche Distanz der Bürger zur Politik“ in Deutschland (2008-06-30).pretty.json\n", + "51446 - Studie: „Große grundsätzliche Distanz der Bürger zur Politik“ in Deutschland (2008-06-30)\n", + "Processing datasets/german-quotations/train/52330 - US-Wahlen: Spitzenduo der Demokraten für die US-Präsidentschaftswahl steht fest (2008-08-24).pretty.json\n", + "52330 - US-Wahlen: Spitzenduo der Demokraten für die US-Präsidentschaftswahl steht fest (2008-08-24)\n", + "Processing datasets/german-quotations/train/83244 - Badeunfälle: Fußballspielerin Florijana Ismaili, 24, Schauspielerin Lisa Martinek, 47, gestorben (2019-07-04).pretty.json\n", + "83244 - Badeunfälle: Fußballspielerin Florijana Ismaili, 24, Schauspielerin Lisa Martinek, 47, gestorben (2019-07-04)\n", + "Processing datasets/german-quotations/train/66494 - Mehrere Dutzend Tote bei Erdbeben in Neuseeland (2011-02-22).pretty.json\n", + "66494 - Mehrere Dutzend Tote bei Erdbeben in Neuseeland (2011-02-22)\n", + "Processing datasets/german-quotations/train/49225 - Nicolas Sarkozys Staatsbesuch im Vereinigten Königreich (2008-03-27).pretty.json\n", + "49225 - Nicolas Sarkozys Staatsbesuch im Vereinigten Königreich (2008-03-27)\n", + "Processing datasets/german-quotations/train/30204 - Zahl der Frauenmorde in Guatemala nimmt weiter zu (2006-07-19).pretty.json\n", + "30204 - Zahl der Frauenmorde in Guatemala nimmt weiter zu (2006-07-19)\n", + "Processing datasets/german-quotations/train/75280 - Nordkorea: Experten im Interview zur aktuellen Situation (0001-01-01).pretty.json\n", + "75280 - Nordkorea: Experten im Interview zur aktuellen Situation (0001-01-01)\n", + "Processing datasets/german-quotations/train/4792 - Der ehemalige ÖTV-Chef, Heinz Kluncker, ist tot (2005-04-22).pretty.json\n", + "4792 - Der ehemalige ÖTV-Chef, Heinz Kluncker, ist tot (2005-04-22)\n", + "Processing datasets/german-quotations/train/2097 - Topmanager verurteilt – 300 Arbeitsplätze weg (2004-12-20).pretty.json\n", + "2097 - Topmanager verurteilt – 300 Arbeitsplätze weg (2004-12-20)\n", + "Processing datasets/german-quotations/train/4867 - Urvögel werden oft fehlgedeutet (2005-04-24).pretty.json\n", + "4867 - Urvögel werden oft fehlgedeutet (2005-04-24)\n", + "Processing datasets/german-quotations/train/70718 - Niedersachsen: CDU-Abgeordneter beendet wegen Internetkontakts zu 15-Jähriger seine politische Karriere (2011-12-09).pretty.json\n", + "70718 - Niedersachsen: CDU-Abgeordneter beendet wegen Internetkontakts zu 15-Jähriger seine politische Karriere (2011-12-09)\n", + "Processing datasets/german-quotations/train/3574 - A1-Ausbau wird privat finanziert (2005-02-27).pretty.json\n", + "3574 - A1-Ausbau wird privat finanziert (2005-02-27)\n", + "Processing datasets/german-quotations/train/59921 - Heidelberg: LKW 20 Meter in die Tiefe gestürzt (2009-12-13).pretty.json\n", + "59921 - Heidelberg: LKW 20 Meter in die Tiefe gestürzt (2009-12-13)\n", + "Processing datasets/german-quotations/train/45213 - FIFA vergibt Fußball-Weltmeisterschaften nach Deutschland und Brasilien (2007-10-30).pretty.json\n", + "45213 - FIFA vergibt Fußball-Weltmeisterschaften nach Deutschland und Brasilien (2007-10-30)\n", + "Processing datasets/german-quotations/train/20800 - Flugpassagier sprang aus rollendem Flugzeug auf die Startbahn (2006-01-26).pretty.json\n", + "20800 - Flugpassagier sprang aus rollendem Flugzeug auf die Startbahn (2006-01-26)\n", + "Processing datasets/german-quotations/train/58108 - Gutachten: John Demjanjuk ist verhandlungsfähig (2009-07-03).pretty.json\n", + "58108 - Gutachten: John Demjanjuk ist verhandlungsfähig (2009-07-03)\n", + "Processing datasets/german-quotations/train/43807 - Elfjähriger auf offener Straße erschossen – mutmaßlicher Mörder auf Kaution entlassen (2007-08-25).pretty.json\n", + "43807 - Elfjähriger auf offener Straße erschossen – mutmaßlicher Mörder auf Kaution entlassen (2007-08-25)\n", + "Processing datasets/german-quotations/train/36639 - Anti-Piraterie-Kopierschutz für hochauflösende Speichermedien möglicherweise gehackt (2006-12-30).pretty.json\n", + "36639 - Anti-Piraterie-Kopierschutz für hochauflösende Speichermedien möglicherweise gehackt (2006-12-30)\n", + "Processing datasets/german-quotations/train/36397 - Stefan Hentschel ist tot (2006-12-19).pretty.json\n", + "36397 - Stefan Hentschel ist tot (2006-12-19)\n", + "Processing datasets/german-quotations/train/46358 - Schweres Zugunglück in Pakistan (2007-12-20).pretty.json\n", + "46358 - Schweres Zugunglück in Pakistan (2007-12-20)\n", + "Processing datasets/german-quotations/train/74557 - Russischer Präsident Putin unterzeichnet Gesetz zu Adoptionsverbot durch US-Bürger (2012-12-30).pretty.json\n", + "74557 - Russischer Präsident Putin unterzeichnet Gesetz zu Adoptionsverbot durch US-Bürger (2012-12-30)\n", + "Processing datasets/german-quotations/train/48591 - Die Hamburger Grünen wollen in Koalitionsverhandlungen mit der CDU eintreten (2008-03-03).pretty.json\n", + "48591 - Die Hamburger Grünen wollen in Koalitionsverhandlungen mit der CDU eintreten (2008-03-03)\n", + "Processing datasets/german-quotations/train/14986 - Aserbaidschan: UNO-Hubschrauber abgestürzt (2005-10-28).pretty.json\n", + "14986 - Aserbaidschan: UNO-Hubschrauber abgestürzt (2005-10-28)\n", + "Processing datasets/german-quotations/train/22508 - Düsseldorf: Randale in LTU-Airbus (2006-02-21).pretty.json\n", + "22508 - Düsseldorf: Randale in LTU-Airbus (2006-02-21)\n", + "Processing datasets/german-quotations/train/44301 - Freiwilliger Produktrückruf von Mars-Süßwaren (2007-09-18).pretty.json\n", + "44301 - Freiwilliger Produktrückruf von Mars-Süßwaren (2007-09-18)\n", + "Processing datasets/german-quotations/train/81518 - Kein Investor für Qoros (2017-08-24).pretty.json\n", + "81518 - Kein Investor für Qoros (2017-08-24)\n", + "Processing datasets/german-quotations/train/67886 - Edelmetallpreise auf Höhenflug (2011-04-30).pretty.json\n", + "67886 - Edelmetallpreise auf Höhenflug (2011-04-30)\n", + "Processing datasets/german-quotations/train/74006 - Georgien: Rücktritte nach Folterskandal (2012-09-21).pretty.json\n", + "74006 - Georgien: Rücktritte nach Folterskandal (2012-09-21)\n", + "Processing datasets/german-quotations/train/70935 - Anonymous hackt sich bei US-Sicherheitsunternehmen ein und bucht 1 Million Dollar für gemeinnützige Zwecke ab (2011-12-26).pretty.json\n", + "70935 - Anonymous hackt sich bei US-Sicherheitsunternehmen ein und bucht 1 Million Dollar für gemeinnützige Zwecke ab (2011-12-26)\n", + "Processing datasets/german-quotations/train/2096 - Hilde Gerg schafft 20. Weltcupsieg in St. Moritz (2004-12-21).pretty.json\n", + "2096 - Hilde Gerg schafft 20. Weltcupsieg in St. Moritz (2004-12-21)\n", + "Processing datasets/german-quotations/train/62459 - Thailand: Premierminister kündigt Auflösung des Parlaments für September an (2010-05-06).pretty.json\n", + "62459 - Thailand: Premierminister kündigt Auflösung des Parlaments für September an (2010-05-06)\n", + "Processing datasets/german-quotations/train/74327 - Ägypten: Mindestens 40 Kinder sterben bei Unfall an einem Bahnübergang (2012-11-17).pretty.json\n", + "74327 - Ägypten: Mindestens 40 Kinder sterben bei Unfall an einem Bahnübergang (2012-11-17)\n", + "Processing datasets/german-quotations/train/45695 - IPCC: Der Klimawandel hat bereits begonnen (2007-11-17).pretty.json\n", + "45695 - IPCC: Der Klimawandel hat bereits begonnen (2007-11-17)\n", + "Processing datasets/german-quotations/train/2240 - Forscher möchten den perfekten Weihnachtsbaum in unser Wohnzimmer stellen (2004-12-24).pretty.json\n", + "2240 - Forscher möchten den perfekten Weihnachtsbaum in unser Wohnzimmer stellen (2004-12-24)\n", + "Processing datasets/german-quotations/train/56965 - Verhandlungsunfähig: John Demjanjuk wird doch nicht ausgeliefert (2009-04-05).pretty.json\n", + "56965 - Verhandlungsunfähig: John Demjanjuk wird doch nicht ausgeliefert (2009-04-05)\n", + "Processing datasets/german-quotations/train/52208 - 75 Jahre Volksempfänger (2008-08-19).pretty.json\n", + "52208 - 75 Jahre Volksempfänger (2008-08-19)\n", + "Processing datasets/german-quotations/train/17705 - Englischsprachige Wikipedia nun größer als die spanische Enzyklopädie ESPASA (2005-12-17).pretty.json\n", + "17705 - Englischsprachige Wikipedia nun größer als die spanische Enzyklopädie ESPASA (2005-12-17)\n", + "Processing datasets/german-quotations/train/71480 - Frankreich: Mélenchon empfiehlt Sarkozy Kandidatur in Deutschland (2012-02-03).pretty.json\n", + "71480 - Frankreich: Mélenchon empfiehlt Sarkozy Kandidatur in Deutschland (2012-02-03)\n", + "Processing datasets/german-quotations/train/55268 - Tesla Motors wird Batterienlieferant für Daimler (2009-01-16).pretty.json\n", + "55268 - Tesla Motors wird Batterienlieferant für Daimler (2009-01-16)\n", + "Processing datasets/german-quotations/train/7537 - NASA gab Starttermin für Raumfähre „Discovery“ bekannt (2005-06-30).pretty.json\n", + "7537 - NASA gab Starttermin für Raumfähre „Discovery“ bekannt (2005-06-30)\n", + "Processing datasets/german-quotations/train/83346 - Vereinigtes Königreich: Boris Johnson neuer britischer Premierminister (2019-07-25).pretty.json\n", + "83346 - Vereinigtes Königreich: Boris Johnson neuer britischer Premierminister (2019-07-25)\n", + "Processing datasets/german-quotations/train/60764 - Iran: Zwei Anhänger der Opposition hingerichtet (2010-01-28).pretty.json\n", + "60764 - Iran: Zwei Anhänger der Opposition hingerichtet (2010-01-28)\n", + "Processing datasets/german-quotations/train/27753 - Kolumbien: Präsident Álvaro Uribe Vélez wiedergewählt (2006-05-31).pretty.json\n", + "27753 - Kolumbien: Präsident Álvaro Uribe Vélez wiedergewählt (2006-05-31)\n", + "Processing datasets/german-quotations/train/8562 - NASA nimmt Spaceshuttle-Flüge mit dem Start der Discovery wieder auf (2005-07-26).pretty.json\n", + "8562 - NASA nimmt Spaceshuttle-Flüge mit dem Start der Discovery wieder auf (2005-07-26)\n", + "Processing datasets/german-quotations/train/49443 - Astronomen entdeckten bislang jüngsten Planeten außerhalb unseres Sonnensystems (2008-04-04).pretty.json\n", + "49443 - Astronomen entdeckten bislang jüngsten Planeten außerhalb unseres Sonnensystems (2008-04-04)\n", + "Processing datasets/german-quotations/train/36202 - Siemens-Affäre: Ex-Vorstand Thomas Ganswindt in Haft (2006-12-13).pretty.json\n", + "36202 - Siemens-Affäre: Ex-Vorstand Thomas Ganswindt in Haft (2006-12-13)\n", + "Processing datasets/german-quotations/train/53657 - Deutschland: Tausende Abiturienten nehmen wegen Studiengebühren kein Studium auf (2008-10-29).pretty.json\n", + "53657 - Deutschland: Tausende Abiturienten nehmen wegen Studiengebühren kein Studium auf (2008-10-29)\n", + "Processing datasets/german-quotations/train/25475 - Selbstmordanschlag in Tel Aviv (2006-04-19).pretty.json\n", + "25475 - Selbstmordanschlag in Tel Aviv (2006-04-19)\n", + "Processing datasets/german-quotations/train/58846 - Im Norden des Jemen sterben zahlreiche Menschen bei einem Luftangriff (2009-09-18).pretty.json\n", + "58846 - Im Norden des Jemen sterben zahlreiche Menschen bei einem Luftangriff (2009-09-18)\n", + "Processing datasets/german-quotations/train/4566 - Luftraum über den USA gesperrt (2005-04-11).pretty.json\n", + "4566 - Luftraum über den USA gesperrt (2005-04-11)\n", + "Processing datasets/german-quotations/train/74430 - PNE Wind aus Cuxhaven - die Aktie des Jahres? (2012-12-08).pretty.json\n", + "74430 - PNE Wind aus Cuxhaven - die Aktie des Jahres? (2012-12-08)\n", + "Processing datasets/german-quotations/train/78858 - Missbrauch durch linke Pädagogik schlimmer als durch Kirche ? (2015-03-21).pretty.json\n", + "78858 - Missbrauch durch linke Pädagogik schlimmer als durch Kirche ? (2015-03-21)\n", + "Processing datasets/german-quotations/train/12809 - Norwegische Studie: Auch leichtes Rauchen kann tödlich sein (2005-09-22).pretty.json\n", + "12809 - Norwegische Studie: Auch leichtes Rauchen kann tödlich sein (2005-09-22)\n", + "Processing datasets/german-quotations/train/53092 - Reinhold Gall: „Polizeigesetz ist von einem Geist der Stärkung der Eingriffsbefugnisse des Staates geprägt“ (2008-08-22).pretty.json\n", + "53092 - Reinhold Gall: „Polizeigesetz ist von einem Geist der Stärkung der Eingriffsbefugnisse des Staates geprägt“ (2008-08-22)\n", + "Processing datasets/german-quotations/train/28391 - WM 2006: Lückenhafte Ticket-Kontrolle ermöglicht Schwarzhandel (2006-06-12).pretty.json\n", + "28391 - WM 2006: Lückenhafte Ticket-Kontrolle ermöglicht Schwarzhandel (2006-06-12)\n", + "Processing datasets/german-quotations/train/65561 - Bundeskartellamt: Strompreise nicht manipuliert (2011-01-13).pretty.json\n", + "65561 - Bundeskartellamt: Strompreise nicht manipuliert (2011-01-13)\n", + "Processing datasets/german-quotations/train/41547 - Second Life: Virtueller Bombenanschlag auf ABC-Hauptquartier (2007-05-26).pretty.json\n", + "41547 - Second Life: Virtueller Bombenanschlag auf ABC-Hauptquartier (2007-05-26)\n", + "Processing datasets/german-quotations/train/2896 - LKW Maut im Elsass? (2005-02-01).pretty.json\n", + "2896 - LKW Maut im Elsass? (2005-02-01)\n", + "Processing datasets/german-quotations/train/74899 - Papst Benedikt XVI. tritt zurück (2013-02-11).pretty.json\n", + "74899 - Papst Benedikt XVI. tritt zurück (2013-02-11)\n", + "Processing datasets/german-quotations/train/79621 - Ostern in Deutschland: Muslime konvertieren zum Christentum (2016-04-03).pretty.json\n", + "79621 - Ostern in Deutschland: Muslime konvertieren zum Christentum (2016-04-03)\n", + "Processing datasets/german-quotations/train/44953 - Schwerer Unfall in Hövelhof: Zug zerteilt Traktor (2007-10-19).pretty.json\n", + "44953 - Schwerer Unfall in Hövelhof: Zug zerteilt Traktor (2007-10-19)\n", + "Processing datasets/german-quotations/train/1265 - IAEO behauptet: Nord-Korea hat Atombomben (2004-12-06).pretty.json\n", + "1265 - IAEO behauptet: Nord-Korea hat Atombomben (2004-12-06)\n", + "Processing datasets/german-quotations/train/12839 - Umstrukturierung führt zu Stellenabbau bei RTL (2005-09-24).pretty.json\n", + "12839 - Umstrukturierung führt zu Stellenabbau bei RTL (2005-09-24)\n", + "Processing datasets/german-quotations/train/80703 - Forscher entdecken 400.000 Jahre alten Schädel (2017-03-15).pretty.json\n", + "80703 - Forscher entdecken 400.000 Jahre alten Schädel (2017-03-15)\n", + "Processing datasets/german-quotations/train/31627 - Viele Tote durch Überschwemmungen in Äthiopien (2006-08-17).pretty.json\n", + "31627 - Viele Tote durch Überschwemmungen in Äthiopien (2006-08-17)\n", + "Processing datasets/german-quotations/train/12491 - Ergebnisse der Bundestagswahl 2005 (2005-09-18).pretty.json\n", + "12491 - Ergebnisse der Bundestagswahl 2005 (2005-09-18)\n", + "Processing datasets/german-quotations/train/58160 - Magdeburg: Auftaktsitzung des Stadtrates (2009-07-07).pretty.json\n", + "58160 - Magdeburg: Auftaktsitzung des Stadtrates (2009-07-07)\n", + "Processing datasets/german-quotations/train/6200 - Wege aus der Haushaltskrise (2005-06-14).pretty.json\n", + "6200 - Wege aus der Haushaltskrise (2005-06-14)\n", + "Processing datasets/german-quotations/train/9679 - Flugzeit von Discovery um einen Tag verlängert (2005-07-31).pretty.json\n", + "9679 - Flugzeit von Discovery um einen Tag verlängert (2005-07-31)\n", + "Processing datasets/german-quotations/train/24294 - Unklarheit über die Festnahme weißrussischer Oppositionspolitiker (2006-03-27).pretty.json\n", + "24294 - Unklarheit über die Festnahme weißrussischer Oppositionspolitiker (2006-03-27)\n", + "Processing datasets/german-quotations/train/73970 - Frankreich will 15.000 Roma zur Rückkehr in ihre Heimatländer bewegen (2012-09-14).pretty.json\n", + "73970 - Frankreich will 15.000 Roma zur Rückkehr in ihre Heimatländer bewegen (2012-09-14)\n", + "Processing datasets/german-quotations/train/80806 - St. Petersburg: Tote bei Explosion in der Metro (2017-04-03).pretty.json\n", + "80806 - St. Petersburg: Tote bei Explosion in der Metro (2017-04-03)\n", + "Processing datasets/german-quotations/train/6203 - 800 Autogastankstellen in Deutschland (2005-06-14).pretty.json\n", + "6203 - 800 Autogastankstellen in Deutschland (2005-06-14)\n", + "Processing datasets/german-quotations/train/64840 - Drogenbandenbekämpfung: Polizei stürmt Armenviertel von Rio de Janeiro (2010-11-27).pretty.json\n", + "64840 - Drogenbandenbekämpfung: Polizei stürmt Armenviertel von Rio de Janeiro (2010-11-27)\n", + "Processing datasets/german-quotations/train/29640 - Berlin: Amokfahrer in psychiatrische Klinik eingeliefert, Beifahrerin frei (2006-07-04).pretty.json\n", + "29640 - Berlin: Amokfahrer in psychiatrische Klinik eingeliefert, Beifahrerin frei (2006-07-04)\n", + "Processing datasets/german-quotations/train/7804 - Bundeskabinett stimmt der Einführung biometrischer Reisepässe zu (2005-07-05).pretty.json\n", + "7804 - Bundeskabinett stimmt der Einführung biometrischer Reisepässe zu (2005-07-05)\n", + "Processing datasets/german-quotations/train/56533 - Zyklon Hamish bedroht Australiens Ostküste (2009-03-10).pretty.json\n", + "56533 - Zyklon Hamish bedroht Australiens Ostküste (2009-03-10)\n", + "Processing datasets/german-quotations/train/35877 - NASA plant permanent bemannte Mondstation (2006-12-05).pretty.json\n", + "35877 - NASA plant permanent bemannte Mondstation (2006-12-05)\n", + "Processing datasets/german-quotations/train/48563 - Bundesligaspiel Cottbus gegen Stuttgart wegen Orkan abgesagt (2008-03-01).pretty.json\n", + "48563 - Bundesligaspiel Cottbus gegen Stuttgart wegen Orkan abgesagt (2008-03-01)\n", + "Processing datasets/german-quotations/train/13554 - Erfolgreiches Abschneiden der deutschen Judoka (2005-10-08).pretty.json\n", + "13554 - Erfolgreiches Abschneiden der deutschen Judoka (2005-10-08)\n", + "Processing datasets/german-quotations/train/75229 - Papst Franziskus und der Richtungswechsel (2013-03-31).pretty.json\n", + "75229 - Papst Franziskus und der Richtungswechsel (2013-03-31)\n", + "Processing datasets/german-quotations/train/73758 - Afghanistan: 17 Besucher einer Feier hingerichtet (2012-08-28).pretty.json\n", + "73758 - Afghanistan: 17 Besucher einer Feier hingerichtet (2012-08-28)\n", + "Processing datasets/german-quotations/train/43098 - Schwere Explosion am Weltraumbahnhof Mojave (2007-07-28).pretty.json\n", + "43098 - Schwere Explosion am Weltraumbahnhof Mojave (2007-07-28)\n", + "Processing datasets/german-quotations/train/83332 - Premiere von Rigoletto in Bregenz (2019-07-23).pretty.json\n", + "83332 - Premiere von Rigoletto in Bregenz (2019-07-23)\n", + "Processing datasets/german-quotations/train/26835 - Europatournee von Bon Jovi in Düsseldorf eröffnet (2006-05-15).pretty.json\n", + "26835 - Europatournee von Bon Jovi in Düsseldorf eröffnet (2006-05-15)\n", + "Processing datasets/german-quotations/train/64095 - Nils Schmid zum SPD-Spitzenkandidaten bei den baden-württembergischen Landtagswahlen gewählt (2010-10-18).pretty.json\n", + "64095 - Nils Schmid zum SPD-Spitzenkandidaten bei den baden-württembergischen Landtagswahlen gewählt (2010-10-18)\n", + "Processing datasets/german-quotations/train/51747 - Deutschland: Linkspartei wegen Nähe zur PKK in der Kritik (2008-07-20).pretty.json\n", + "51747 - Deutschland: Linkspartei wegen Nähe zur PKK in der Kritik (2008-07-20)\n", + "Processing datasets/german-quotations/train/46721 - Randale und Unfälle an Silvester (2008-01-02).pretty.json\n", + "46721 - Randale und Unfälle an Silvester (2008-01-02)\n", + "Processing datasets/german-quotations/train/71492 - Deutschland muss Nazi-Opfern keine Entschädigung zahlen (2012-02-03).pretty.json\n", + "71492 - Deutschland muss Nazi-Opfern keine Entschädigung zahlen (2012-02-03)\n", + "Processing datasets/german-quotations/train/52789 - Neun Jahre altes Hühnerfleisch beschlagnahmt (2008-09-10).pretty.json\n", + "52789 - Neun Jahre altes Hühnerfleisch beschlagnahmt (2008-09-10)\n", + "Processing datasets/german-quotations/train/13443 - NHL: Nach Lockout nun erster Spieltag in der Saison 2005-06 (2005-10-06).pretty.json\n", + "13443 - NHL: Nach Lockout nun erster Spieltag in der Saison 2005-06 (2005-10-06)\n", + "Processing datasets/german-quotations/train/59630 - Fußballnationaltorwart Robert Enke tot – Polizei spricht von Suizid (2009-11-11).pretty.json\n", + "59630 - Fußballnationaltorwart Robert Enke tot – Polizei spricht von Suizid (2009-11-11)\n", + "Processing datasets/german-quotations/train/52120 - Mindestens 18 Tote bei Bombenanschlag im Libanon (2008-08-13).pretty.json\n", + "52120 - Mindestens 18 Tote bei Bombenanschlag im Libanon (2008-08-13)\n", + "Processing datasets/german-quotations/train/64689 - Flugsicherheit: Mutmaßliche Paketbombe auf dem Flug Windhoek - München war Attrappe (2010-11-22).pretty.json\n", + "64689 - Flugsicherheit: Mutmaßliche Paketbombe auf dem Flug Windhoek - München war Attrappe (2010-11-22)\n", + "Processing datasets/german-quotations/train/3552 - Wahlen im Kanton Solothurn (2005-02-27).pretty.json\n", + "3552 - Wahlen im Kanton Solothurn (2005-02-27)\n", + "Processing datasets/german-quotations/train/45274 - Gold so teuer wie schon lange nicht mehr (2007-11-01).pretty.json\n", + "45274 - Gold so teuer wie schon lange nicht mehr (2007-11-01)\n", + "Processing datasets/german-quotations/train/69120 - Israelische Soldaten überwältigen 16 pro-palästinensische Aktivisten (2011-07-20).pretty.json\n", + "69120 - Israelische Soldaten überwältigen 16 pro-palästinensische Aktivisten (2011-07-20)\n", + "Processing datasets/german-quotations/train/54978 - Tag 5 der Angriffe auf Gaza – Israel lehnt Waffenstillstand vorerst ab (2008-12-31).pretty.json\n", + "54978 - Tag 5 der Angriffe auf Gaza – Israel lehnt Waffenstillstand vorerst ab (2008-12-31)\n", + "Processing datasets/german-quotations/train/21932 - Elterninitiative „Vermisste Kinder“ fand Kind nach sechs Jahren (2006-02-12).pretty.json\n", + "21932 - Elterninitiative „Vermisste Kinder“ fand Kind nach sechs Jahren (2006-02-12)\n", + "Processing datasets/german-quotations/train/50554 - EADS: Airbus-Werk in Laupheim soll an Diehl-Thales verkauft werden (2008-05-31).pretty.json\n", + "50554 - EADS: Airbus-Werk in Laupheim soll an Diehl-Thales verkauft werden (2008-05-31)\n", + "Processing datasets/german-quotations/train/69598 - Osram: Preisanstieg bei Seltenen Erden macht Energiesparlampen teuer (2011-08-31).pretty.json\n", + "69598 - Osram: Preisanstieg bei Seltenen Erden macht Energiesparlampen teuer (2011-08-31)\n", + "Processing datasets/german-quotations/train/4833 - Neuer Zwischenfall an gefährlicher Bahnschranke (2005-04-23).pretty.json\n", + "4833 - Neuer Zwischenfall an gefährlicher Bahnschranke (2005-04-23)\n", + "Processing datasets/german-quotations/train/77041 - Islamische Missionierung via SMS (2014-07-03).pretty.json\n", + "77041 - Islamische Missionierung via SMS (2014-07-03)\n", + "Processing datasets/german-quotations/train/12716 - Absturz der Su-27: Russland erwartet von Litauen Einhaltung des Völkerrechts (2005-09-20).pretty.json\n", + "12716 - Absturz der Su-27: Russland erwartet von Litauen Einhaltung des Völkerrechts (2005-09-20)\n", + "Processing datasets/german-quotations/train/84342 - Früherer US-Außenminister Shultz ist tot (2021-02-09).pretty.json\n", + "84342 - Früherer US-Außenminister Shultz ist tot (2021-02-09)\n", + "Processing datasets/german-quotations/train/38177 - Kreis Kusel: Um Zigaretten gebeten und verprügelt (2007-02-14).pretty.json\n", + "38177 - Kreis Kusel: Um Zigaretten gebeten und verprügelt (2007-02-14)\n", + "Processing datasets/german-quotations/train/80902 - Nach Explosionen beim BVB-Bus: Polizei geht von terroristischem Motiv aus (2017-04-12).pretty.json\n", + "80902 - Nach Explosionen beim BVB-Bus: Polizei geht von terroristischem Motiv aus (2017-04-12)\n", + "Processing datasets/german-quotations/train/77697 - OPEC unter Druck : Ölpreis fällt weiter (2014-11-28).pretty.json\n", + "77697 - OPEC unter Druck : Ölpreis fällt weiter (2014-11-28)\n", + "Processing datasets/german-quotations/train/36600 - Eisbären sollen in den USA auf die Liste bedrohter Tierarten (2006-12-28).pretty.json\n", + "36600 - Eisbären sollen in den USA auf die Liste bedrohter Tierarten (2006-12-28)\n", + "Processing datasets/german-quotations/train/70721 - Pieper bittet nach Entgleisung um Entschuldigung (2011-12-09).pretty.json\n", + "70721 - Pieper bittet nach Entgleisung um Entschuldigung (2011-12-09)\n", + "Processing datasets/german-quotations/train/69813 - NASA-Forschungssatellit stürzt auf die Erde (2011-09-23).pretty.json\n", + "69813 - NASA-Forschungssatellit stürzt auf die Erde (2011-09-23)\n", + "Processing datasets/german-quotations/train/37530 - „Münstersche Zeitung“: Über Nacht komplette Redaktion ausgetauscht (2007-01-25).pretty.json\n", + "37530 - „Münstersche Zeitung“: Über Nacht komplette Redaktion ausgetauscht (2007-01-25)\n", + "Processing datasets/german-quotations/train/52796 - OSZE kritisiert Verlauf der Parlamentswahl in Georgien (2008-09-10).pretty.json\n", + "52796 - OSZE kritisiert Verlauf der Parlamentswahl in Georgien (2008-09-10)\n", + "Processing datasets/german-quotations/train/15794 - Dritte Nacht in Folge: Brandstiftungen in Bremen (2005-11-09).pretty.json\n", + "15794 - Dritte Nacht in Folge: Brandstiftungen in Bremen (2005-11-09)\n", + "Processing datasets/german-quotations/train/74342 - Expansion: DB übernimmt Busverkehr in Budapest (2012-11-30).pretty.json\n", + "74342 - Expansion: DB übernimmt Busverkehr in Budapest (2012-11-30)\n", + "Processing datasets/german-quotations/train/75759 - Loeschcontainer-Prism-Spähprogramms (2013-06-18).pretty.json\n", + "75759 - Loeschcontainer-Prism-Spähprogramms (2013-06-18)\n", + "Processing datasets/german-quotations/train/33494 - Mörder von Jennifer vor Gericht (2006-10-03).pretty.json\n", + "33494 - Mörder von Jennifer vor Gericht (2006-10-03)\n", + "Processing datasets/german-quotations/train/70915 - Ukraine: Berufungsgericht bestätigt hohe Haftstrafe für Timoschenko (2011-12-25).pretty.json\n", + "70915 - Ukraine: Berufungsgericht bestätigt hohe Haftstrafe für Timoschenko (2011-12-25)\n", + "Processing datasets/german-quotations/train/47722 - Schusswechsel an der israelisch-libanesischen Grenze (2008-02-04).pretty.json\n", + "47722 - Schusswechsel an der israelisch-libanesischen Grenze (2008-02-04)\n", + "Processing datasets/german-quotations/train/1959 - Ice Tigers an der Spitze (2004-12-17).pretty.json\n", + "1959 - Ice Tigers an der Spitze (2004-12-17)\n", + "Processing datasets/german-quotations/train/81786 - München-Hauptbahnhof: Sachse mit 6,52 Promille in Gewahrsam genommen (2017-10-18).pretty.json\n", + "81786 - München-Hauptbahnhof: Sachse mit 6,52 Promille in Gewahrsam genommen (2017-10-18)\n", + "Processing datasets/german-quotations/train/5456 - Türkische Chartermaschine musste auf dem Flughafen von Tel Aviv notlanden (2005-05-20).pretty.json\n", + "5456 - Türkische Chartermaschine musste auf dem Flughafen von Tel Aviv notlanden (2005-05-20)\n", + "Processing datasets/german-quotations/train/50016 - Niedersachsen: Proteste gegen Verordnungsentwurf zum Lehrerarbeitszeitkonto erfolgreich (2008-05-08).pretty.json\n", + "50016 - Niedersachsen: Proteste gegen Verordnungsentwurf zum Lehrerarbeitszeitkonto erfolgreich (2008-05-08)\n", + "Processing datasets/german-quotations/train/45124 - Schweres Unglück im Braunkohle-Kraftwerk Neurath (2007-10-26).pretty.json\n", + "45124 - Schweres Unglück im Braunkohle-Kraftwerk Neurath (2007-10-26)\n", + "Processing datasets/german-quotations/train/35012 - Pfaff entlässt bis zu 130 Mitarbeiter (2006-11-15).pretty.json\n", + "35012 - Pfaff entlässt bis zu 130 Mitarbeiter (2006-11-15)\n", + "Processing datasets/german-quotations/train/64109 - Eisenbahnerstreik in Belgien (2010-10-18).pretty.json\n", + "64109 - Eisenbahnerstreik in Belgien (2010-10-18)\n", + "Processing datasets/german-quotations/train/3814 - Umsatzeinbruch bei KarstadtQuelle zu Jahresbeginn (2005-03-09).pretty.json\n", + "3814 - Umsatzeinbruch bei KarstadtQuelle zu Jahresbeginn (2005-03-09)\n", + "Processing datasets/german-quotations/train/55189 - Somalische Piraten ertrinken mit Lösegeld (2009-01-12).pretty.json\n", + "55189 - Somalische Piraten ertrinken mit Lösegeld (2009-01-12)\n", + "Processing datasets/german-quotations/train/79034 - Chefredakteur der Schweizer Sonntagszeitung schreckt deutsche Medien auf (2015-05-27).pretty.json\n", + "79034 - Chefredakteur der Schweizer Sonntagszeitung schreckt deutsche Medien auf (2015-05-27)\n", + "Processing datasets/german-quotations/train/18979 - Schalke 04 hat einen neuen Trainer (2006-01-04).pretty.json\n", + "18979 - Schalke 04 hat einen neuen Trainer (2006-01-04)\n", + "Processing datasets/german-quotations/train/27782 - Weltnichtrauchertag 2006 (2006-06-09).pretty.json\n", + "27782 - Weltnichtrauchertag 2006 (2006-06-09)\n", + "Processing datasets/german-quotations/train/77730 - Leerer Koffer sorgt für Großeinsatz am Praterstern (2014-12-02).pretty.json\n", + "77730 - Leerer Koffer sorgt für Großeinsatz am Praterstern (2014-12-02)\n", + "Processing datasets/german-quotations/train/58907 - Polizei räumt Flüchtlingslager bei Calais (2009-09-22).pretty.json\n", + "58907 - Polizei räumt Flüchtlingslager bei Calais (2009-09-22)\n", + "Processing datasets/german-quotations/train/13726 - Seeschnecken: Sex ist eine Kostenfrage (2005-10-11).pretty.json\n", + "13726 - Seeschnecken: Sex ist eine Kostenfrage (2005-10-11)\n", + "Processing datasets/german-quotations/train/12438 - Olympiaturm in München nach Bombendrohung geräumt (2005-09-13).pretty.json\n", + "12438 - Olympiaturm in München nach Bombendrohung geräumt (2005-09-13)\n", + "Processing datasets/german-quotations/train/33262 - Thailands Militärjunta will nach Machtübergabe weiter Einfluss ausüben (2006-09-26).pretty.json\n", + "33262 - Thailands Militärjunta will nach Machtübergabe weiter Einfluss ausüben (2006-09-26)\n", + "Processing datasets/german-quotations/train/833 - 40. Jahrestag der Gründung von Stiftung Warentest (2004-12-04).pretty.json\n", + "833 - 40. Jahrestag der Gründung von Stiftung Warentest (2004-12-04)\n", + "Processing datasets/german-quotations/train/55796 - Wiener Linien – Warum sollten wir zahlen? (2009-02-07).pretty.json\n", + "55796 - Wiener Linien – Warum sollten wir zahlen? (2009-02-07)\n", + "Processing datasets/german-quotations/train/54578 - Kürzung der Pendlerpauschale ist verfassungswidrig (2008-12-09).pretty.json\n", + "54578 - Kürzung der Pendlerpauschale ist verfassungswidrig (2008-12-09)\n", + "Processing datasets/german-quotations/train/5943 - Kein EU-Referendum in Großbritannien (2005-06-06).pretty.json\n", + "5943 - Kein EU-Referendum in Großbritannien (2005-06-06)\n", + "Processing datasets/german-quotations/train/83439 - Erdrutsch in Chile: Mindestens sechs Tote bei Einsturz von zwei Häusern (2019-08-20).pretty.json\n", + "83439 - Erdrutsch in Chile: Mindestens sechs Tote bei Einsturz von zwei Häusern (2019-08-20)\n", + "Processing datasets/german-quotations/train/3212 - Hans-Joachim Watzke wird neuer Geschäftsführer von Borussia Dortmund (2005-02-15).pretty.json\n", + "3212 - Hans-Joachim Watzke wird neuer Geschäftsführer von Borussia Dortmund (2005-02-15)\n", + "Processing datasets/german-quotations/train/3681 - Riesiges Ozonloch über der nördlichen Erdkugel entdeckt (2005-03-03).pretty.json\n", + "3681 - Riesiges Ozonloch über der nördlichen Erdkugel entdeckt (2005-03-03)\n", + "Processing datasets/german-quotations/train/5014 - Weltgrößtes Containerschiff in Hamburg eingetroffen (2005-04-30).pretty.json\n", + "5014 - Weltgrößtes Containerschiff in Hamburg eingetroffen (2005-04-30)\n", + "Processing datasets/german-quotations/train/34501 - Lager von „Genreis“ der Sorte LL601 im Hamburger Hafen entdeckt (2006-10-30).pretty.json\n", + "34501 - Lager von „Genreis“ der Sorte LL601 im Hamburger Hafen entdeckt (2006-10-30)\n", + "Processing datasets/german-quotations/train/10367 - Franka Dietzsch wird Weltmeisterin im Diskuswerfen (2005-08-11).pretty.json\n", + "10367 - Franka Dietzsch wird Weltmeisterin im Diskuswerfen (2005-08-11)\n", + "Processing datasets/german-quotations/train/63107 - Fußball-WM: Kameruns Ausscheiden steht bereits fest (2010-06-27).pretty.json\n", + "63107 - Fußball-WM: Kameruns Ausscheiden steht bereits fest (2010-06-27)\n", + "Processing datasets/german-quotations/train/14183 - Saddam-Prozess in Bagdad begonnen (2005-10-19).pretty.json\n", + "14183 - Saddam-Prozess in Bagdad begonnen (2005-10-19)\n", + "Processing datasets/german-quotations/train/17789 - Sorge um Gedenktafel für abgestürzten „Rosinenbomber“ (2005-12-18).pretty.json\n", + "17789 - Sorge um Gedenktafel für abgestürzten „Rosinenbomber“ (2005-12-18)\n", + "Processing datasets/german-quotations/train/82411 - Polizei erhöht Kontrolldruck in Kaiserslauterer Innenstadt (2018-05-13).pretty.json\n", + "82411 - Polizei erhöht Kontrolldruck in Kaiserslauterer Innenstadt (2018-05-13)\n", + "Processing datasets/german-quotations/train/40316 - Thailand: König begnadigt wegen Majestätsbeleidigung verurteilten Schweizer (2007-04-12).pretty.json\n", + "40316 - Thailand: König begnadigt wegen Majestätsbeleidigung verurteilten Schweizer (2007-04-12)\n", + "Processing datasets/german-quotations/train/52199 - Pakistans Präsident Musharraf kündigt Rücktritt an (2008-08-18).pretty.json\n", + "52199 - Pakistans Präsident Musharraf kündigt Rücktritt an (2008-08-18)\n", + "Processing datasets/german-quotations/train/60138 - China: Fünf Todesurteile wegen Juli-Unruhen in der Provinz Xinjiang (2009-12-24).pretty.json\n", + "60138 - China: Fünf Todesurteile wegen Juli-Unruhen in der Provinz Xinjiang (2009-12-24)\n", + "Processing datasets/german-quotations/train/77719 - Hannover, Heidelberg und Mannheim sind jetzt „UNESCO Creative Cities“ (2014-12-03).pretty.json\n", + "77719 - Hannover, Heidelberg und Mannheim sind jetzt „UNESCO Creative Cities“ (2014-12-03)\n", + "Processing datasets/german-quotations/train/42688 - Al Qaida droht Vergeltungsschläge für Rushdie-Ehrung an (2007-07-11).pretty.json\n", + "42688 - Al Qaida droht Vergeltungsschläge für Rushdie-Ehrung an (2007-07-11)\n", + "Processing datasets/german-quotations/train/30570 - Echtheit des Wracks von „Graf Zeppelin“ bestätigt (2006-07-28).pretty.json\n", + "30570 - Echtheit des Wracks von „Graf Zeppelin“ bestätigt (2006-07-28)\n", + "Processing datasets/german-quotations/train/55554 - Kirsten Gillibrand zur Nachfolgerin Hillary Clintons als Senatorin ernannt (2009-01-26).pretty.json\n", + "55554 - Kirsten Gillibrand zur Nachfolgerin Hillary Clintons als Senatorin ernannt (2009-01-26)\n", + "Processing datasets/german-quotations/train/6392 - Florida: Notlandung auf dreispuriger Straße (2005-06-17).pretty.json\n", + "6392 - Florida: Notlandung auf dreispuriger Straße (2005-06-17)\n", + "Processing datasets/german-quotations/train/28750 - New Orleans Bürgermeister forderte Nationalgarde zur Unterstützung an (2006-06-22).pretty.json\n", + "28750 - New Orleans Bürgermeister forderte Nationalgarde zur Unterstützung an (2006-06-22)\n", + "Processing datasets/german-quotations/train/72173 - Polen: Erster Auslandsbesuch Gaucks (2012-03-31).pretty.json\n", + "72173 - Polen: Erster Auslandsbesuch Gaucks (2012-03-31)\n", + "Processing datasets/german-quotations/train/21465 - Dokumentarfilm versucht, Licht in das Geheimnis des Bermudadreiecks zu bringen (2006-02-05).pretty.json\n", + "21465 - Dokumentarfilm versucht, Licht in das Geheimnis des Bermudadreiecks zu bringen (2006-02-05)\n", + "Processing datasets/german-quotations/train/82713 - Merkel kündigt ihren Abschied als CDU-Chefin an (2018-10-31).pretty.json\n", + "82713 - Merkel kündigt ihren Abschied als CDU-Chefin an (2018-10-31)\n", + "Processing datasets/german-quotations/train/5954 - Pride of America mit Verspätung an Reederei übergeben (2005-06-07).pretty.json\n", + "5954 - Pride of America mit Verspätung an Reederei übergeben (2005-06-07)\n", + "Processing datasets/german-quotations/train/68296 - Situation vor Unabhängigkeit Südsudans gespannt – Kämpfe in Ölregion (2011-05-23).pretty.json\n", + "68296 - Situation vor Unabhängigkeit Südsudans gespannt – Kämpfe in Ölregion (2011-05-23)\n", + "Processing datasets/german-quotations/train/40144 - WHO empfiehlt Beschneidung von Männern (2007-04-08).pretty.json\n", + "40144 - WHO empfiehlt Beschneidung von Männern (2007-04-08)\n", + "Processing datasets/german-quotations/train/49483 - Olympischer Fackellauf in Paris endet im Chaos (2008-04-08).pretty.json\n", + "49483 - Olympischer Fackellauf in Paris endet im Chaos (2008-04-08)\n", + "Processing datasets/german-quotations/train/54361 - Mehrheit der Grönländer spricht sich für mehr Unabhängigkeit von Dänemark aus (2008-11-27).pretty.json\n", + "54361 - Mehrheit der Grönländer spricht sich für mehr Unabhängigkeit von Dänemark aus (2008-11-27)\n", + "Processing datasets/german-quotations/train/62831 - England: Taxifahrer richtet Blutbad an – 13 Tote (2010-06-03).pretty.json\n", + "62831 - England: Taxifahrer richtet Blutbad an – 13 Tote (2010-06-03)\n", + "Processing datasets/german-quotations/train/75684 - Fairphone: Nachhaltiges Smartphone geht in Produktion (2013-06-08).pretty.json\n", + "75684 - Fairphone: Nachhaltiges Smartphone geht in Produktion (2013-06-08)\n", + "Processing datasets/german-quotations/train/48246 - EU will illegale Downloads mit kommerziellen Absichten unter Strafe stellen (2008-02-23).pretty.json\n", + "48246 - EU will illegale Downloads mit kommerziellen Absichten unter Strafe stellen (2008-02-23)\n", + "Processing datasets/german-quotations/train/80961 - Anschlag auf Buskonvoi in Syrien international geächtet (2017-04-17).pretty.json\n", + "80961 - Anschlag auf Buskonvoi in Syrien international geächtet (2017-04-17)\n", + "Processing datasets/german-quotations/train/50164 - Myanmar: Die Arbeit von Hilfsorganisationen wird weiter behindert (2008-05-14).pretty.json\n", + "50164 - Myanmar: Die Arbeit von Hilfsorganisationen wird weiter behindert (2008-05-14)\n", + "Processing datasets/german-quotations/train/48690 - Deutschland: Streiks im öffentlichen Dienst – Mittwoch auch Flughäfen betroffen (2008-03-07).pretty.json\n", + "48690 - Deutschland: Streiks im öffentlichen Dienst – Mittwoch auch Flughäfen betroffen (2008-03-07)\n", + "Processing datasets/german-quotations/train/79138 - Großbritannien plant Sperrung von Internetdiensten (2015-07-10).pretty.json\n", + "79138 - Großbritannien plant Sperrung von Internetdiensten (2015-07-10)\n", + "Processing datasets/german-quotations/train/66967 - London: Alkoholentzug eines Dreijährigen (2011-03-15).pretty.json\n", + "66967 - London: Alkoholentzug eines Dreijährigen (2011-03-15)\n", + "Processing datasets/german-quotations/train/40335 - Kritik am geplanten Online-Zugriff der Polizei auf digitalisierte Passfotos (2007-04-12).pretty.json\n", + "40335 - Kritik am geplanten Online-Zugriff der Polizei auf digitalisierte Passfotos (2007-04-12)\n", + "Processing datasets/german-quotations/train/60990 - 130. Shuttle Mission: Raumfähre Endeavour zur Internationalen Raumstation gestartet (2010-02-08).pretty.json\n", + "60990 - 130. Shuttle Mission: Raumfähre Endeavour zur Internationalen Raumstation gestartet (2010-02-08)\n", + "Processing datasets/german-quotations/train/15578 - Zwölfter Spieltag der Fußball-Bundesligasaison 2005-06 (2005-11-05).pretty.json\n", + "15578 - Zwölfter Spieltag der Fußball-Bundesligasaison 2005-06 (2005-11-05)\n", + "Processing datasets/german-quotations/train/71723 - Verdeckte Ermittler des FBI verhindern Anschlag auf das Kapitol (2012-02-19).pretty.json\n", + "71723 - Verdeckte Ermittler des FBI verhindern Anschlag auf das Kapitol (2012-02-19)\n", + "Processing datasets/german-quotations/train/51344 - 34-jähriger Balver wegen versuchten Totschlags in Untersuchungshaft (2008-06-28).pretty.json\n", + "51344 - 34-jähriger Balver wegen versuchten Totschlags in Untersuchungshaft (2008-06-28)\n", + "Processing datasets/german-quotations/train/37427 - Unbekannter besucht jährlich Edgar Allan Poes Grab (2007-01-22).pretty.json\n", + "37427 - Unbekannter besucht jährlich Edgar Allan Poes Grab (2007-01-22)\n", + "Processing datasets/german-quotations/train/73781 - Wurde Jassir Arafat vergiftet? Französische Justiz ermittelt (2012-09-02).pretty.json\n", + "73781 - Wurde Jassir Arafat vergiftet? Französische Justiz ermittelt (2012-09-02)\n", + "Processing datasets/german-quotations/train/51963 - Hamburg: Mitglieder der GRÜNEN zur ödp gewechselt (2008-08-07).pretty.json\n", + "51963 - Hamburg: Mitglieder der GRÜNEN zur ödp gewechselt (2008-08-07)\n", + "Processing datasets/german-quotations/train/21815 - Steve Fossett: Benzinverlust beim Start des Rekordflugversuchs beunruhigt Kontrollpersonal (2006-02-10).pretty.json\n", + "21815 - Steve Fossett: Benzinverlust beim Start des Rekordflugversuchs beunruhigt Kontrollpersonal (2006-02-10)\n", + "Processing datasets/german-quotations/train/40724 - Phil Spector wegen Mordes vor Gericht (2007-05-01).pretty.json\n", + "40724 - Phil Spector wegen Mordes vor Gericht (2007-05-01)\n", + "Processing datasets/german-quotations/train/5248 - Charterflugzeug in Australien abgestürzt (2005-05-07).pretty.json\n", + "5248 - Charterflugzeug in Australien abgestürzt (2005-05-07)\n", + "Processing datasets/german-quotations/train/60485 - Kartellamt in Deutschland: Erneut Razzien wegen des Verdachts auf illegale Preisabsprachen (2010-01-14).pretty.json\n", + "60485 - Kartellamt in Deutschland: Erneut Razzien wegen des Verdachts auf illegale Preisabsprachen (2010-01-14)\n", + "Processing datasets/german-quotations/train/33957 - Türkischer Hochschulratspräsident gab französische Auszeichnung zurück (2006-10-17).pretty.json\n", + "33957 - Türkischer Hochschulratspräsident gab französische Auszeichnung zurück (2006-10-17)\n", + "Processing datasets/german-quotations/train/20089 - Hamburg: Betreiberwechsel in der Luftrettung (2006-01-15).pretty.json\n", + "20089 - Hamburg: Betreiberwechsel in der Luftrettung (2006-01-15)\n", + "Processing datasets/german-quotations/train/30965 - Überschwemmungen in Äthiopien fordern mehr als 100 Tote (2006-08-07).pretty.json\n", + "30965 - Überschwemmungen in Äthiopien fordern mehr als 100 Tote (2006-08-07)\n", + "Processing datasets/german-quotations/train/3434 - Giftmüllskandal in Basel? (2005-02-24).pretty.json\n", + "3434 - Giftmüllskandal in Basel? (2005-02-24)\n", + "Processing datasets/german-quotations/train/4430 - Neuer Parlamentspräsident im Irak (2005-04-03).pretty.json\n", + "4430 - Neuer Parlamentspräsident im Irak (2005-04-03)\n", + "Processing datasets/german-quotations/train/13100 - Laut Stiftung Warentest sind viele alternative Heilverfahren unwirksam (2005-09-29).pretty.json\n", + "13100 - Laut Stiftung Warentest sind viele alternative Heilverfahren unwirksam (2005-09-29)\n", + "Processing datasets/german-quotations/train/2840 - Papier hält NPD-Verbot für möglich (2005-01-29).pretty.json\n", + "2840 - Papier hält NPD-Verbot für möglich (2005-01-29)\n", + "Processing datasets/german-quotations/train/71200 - Rolls Royce bricht Verkaufsrekord (2012-01-10).pretty.json\n", + "71200 - Rolls Royce bricht Verkaufsrekord (2012-01-10)\n", + "Processing datasets/german-quotations/train/43535 - MSV Duisburg siegt in Dortmund mit 3:1 (2007-08-13).pretty.json\n", + "43535 - MSV Duisburg siegt in Dortmund mit 3:1 (2007-08-13)\n", + "Processing datasets/german-quotations/train/56876 - Mehrere Tote bei Amoklauf im Pinelake-Altersheim in Carthage (2009-03-30).pretty.json\n", + "56876 - Mehrere Tote bei Amoklauf im Pinelake-Altersheim in Carthage (2009-03-30)\n", + "Processing datasets/german-quotations/train/48539 - Mehr als 1.000 Messer in Laguiole mutmaßlich gestohlen (2008-02-29).pretty.json\n", + "48539 - Mehr als 1.000 Messer in Laguiole mutmaßlich gestohlen (2008-02-29)\n", + "Processing datasets/german-quotations/train/64977 - Deutschland: Gewerkschaften kündigen Warnstreiks gegen RWE an (2010-12-05).pretty.json\n", + "64977 - Deutschland: Gewerkschaften kündigen Warnstreiks gegen RWE an (2010-12-05)\n", + "Processing datasets/german-quotations/train/63385 - Flugzeugabsturz in Alaska fordert fünf Opfer (2010-08-11).pretty.json\n", + "63385 - Flugzeugabsturz in Alaska fordert fünf Opfer (2010-08-11)\n", + "Processing datasets/german-quotations/train/70140 - Gaddafi in der Sahara beerdigt (2011-10-25).pretty.json\n", + "70140 - Gaddafi in der Sahara beerdigt (2011-10-25)\n", + "Processing datasets/german-quotations/train/39262 - Lateinamerikanische Migranten sandten im letzten Jahr 62 Milliarden Dollar in ihre Heimat (2007-03-19).pretty.json\n", + "39262 - Lateinamerikanische Migranten sandten im letzten Jahr 62 Milliarden Dollar in ihre Heimat (2007-03-19)\n", + "Processing datasets/german-quotations/train/56063 - Designierter Linzer Weihbischof Wagner verzichtet auf Amt (2009-02-16).pretty.json\n", + "56063 - Designierter Linzer Weihbischof Wagner verzichtet auf Amt (2009-02-16)\n", + "Processing datasets/german-quotations/train/71593 - Alstereisvergnügen in Hamburg lockt zahlreiche Besucher an (2012-02-14).pretty.json\n", + "71593 - Alstereisvergnügen in Hamburg lockt zahlreiche Besucher an (2012-02-14)\n", + "Processing datasets/german-quotations/train/73882 - Ungarn: Roma-Gardechef Ferenc Bago festgenommen (2012-09-07).pretty.json\n", + "73882 - Ungarn: Roma-Gardechef Ferenc Bago festgenommen (2012-09-07)\n", + "Processing datasets/german-quotations/train/38180 - EU-Gerichtsgutachter: VW-Gesetz verstößt gegen Europarecht (2007-02-13).pretty.json\n", + "38180 - EU-Gerichtsgutachter: VW-Gesetz verstößt gegen Europarecht (2007-02-13)\n", + "Processing datasets/german-quotations/train/9837 - Adidas kauft Reebok (2005-08-03).pretty.json\n", + "9837 - Adidas kauft Reebok (2005-08-03)\n", + "Processing datasets/german-quotations/train/40419 - Frankfurt: Mindestens eintausend Menschen demonstrieren gegen den „Überwachungswahn“ (2007-04-16).pretty.json\n", + "40419 - Frankfurt: Mindestens eintausend Menschen demonstrieren gegen den „Überwachungswahn“ (2007-04-16)\n", + "Processing datasets/german-quotations/train/46350 - 26-Jähriger überfuhr Fahrradfahrer in München: 18 Monate Haft (2007-12-20).pretty.json\n", + "46350 - 26-Jähriger überfuhr Fahrradfahrer in München: 18 Monate Haft (2007-12-20)\n", + "Processing datasets/german-quotations/train/64227 - USA: Republikaner wollen Subventionen für öffentlichen Rundfunk streichen (2010-10-23).pretty.json\n", + "64227 - USA: Republikaner wollen Subventionen für öffentlichen Rundfunk streichen (2010-10-23)\n", + "Processing datasets/german-quotations/train/14002 - Rumänien: Verdacht auf Vogelgrippe bestätigt (2005-10-16).pretty.json\n", + "14002 - Rumänien: Verdacht auf Vogelgrippe bestätigt (2005-10-16)\n", + "Processing datasets/german-quotations/train/52082 - Eröffnungsfeier in Peking offenbar manipuliert (2008-08-13).pretty.json\n", + "52082 - Eröffnungsfeier in Peking offenbar manipuliert (2008-08-13)\n", + "Processing datasets/german-quotations/train/74210 - Erinnerungsstätte an Hexenprozesse in Schottland (2012-10-30).pretty.json\n", + "74210 - Erinnerungsstätte an Hexenprozesse in Schottland (2012-10-30)\n", + "Processing datasets/german-quotations/train/22808 - Jean Todt: Der Ferrari-Teamchef wurde 60 (2006-03-02).pretty.json\n", + "22808 - Jean Todt: Der Ferrari-Teamchef wurde 60 (2006-03-02)\n", + "Processing datasets/german-quotations/train/1034 - BGH erlässt Haftbefehl gegen drei mutmaßliche irakische Terroristen (2004-12-05).pretty.json\n", + "1034 - BGH erlässt Haftbefehl gegen drei mutmaßliche irakische Terroristen (2004-12-05)\n", + "Processing datasets/german-quotations/train/60146 - Studenten erhöhen Rekordsumme bei weihnachtlicher Benefizveranstaltung (2009-12-25).pretty.json\n", + "60146 - Studenten erhöhen Rekordsumme bei weihnachtlicher Benefizveranstaltung (2009-12-25)\n", + "Processing datasets/german-quotations/train/1229 - Geiselnahme in Saudi-Arabien (2004-12-06).pretty.json\n", + "1229 - Geiselnahme in Saudi-Arabien (2004-12-06)\n", + "Processing datasets/german-quotations/train/14704 - Michael Ballack ist Kandidat bei der Wahl zu Europas Fußballer des Jahres (2005-10-25).pretty.json\n", + "14704 - Michael Ballack ist Kandidat bei der Wahl zu Europas Fußballer des Jahres (2005-10-25)\n", + "Processing datasets/german-quotations/train/27912 - Weltkonferenz im Ausland lebender Griechen in Athen (2006-06-02).pretty.json\n", + "27912 - Weltkonferenz im Ausland lebender Griechen in Athen (2006-06-02)\n", + "Processing datasets/german-quotations/train/9019 - Horst Köhler lässt den Bundestag auflösen (2005-07-21).pretty.json\n", + "9019 - Horst Köhler lässt den Bundestag auflösen (2005-07-21)\n", + "Processing datasets/german-quotations/train/2464 - Emissionshandel in der Europäischen Union beginnt (2005-01-01).pretty.json\n", + "2464 - Emissionshandel in der Europäischen Union beginnt (2005-01-01)\n", + "Processing datasets/german-quotations/train/75575 - Fracking gefährdet Wasserversorgung der Bierbrauer (2013-05-30).pretty.json\n", + "75575 - Fracking gefährdet Wasserversorgung der Bierbrauer (2013-05-30)\n", + "Processing datasets/german-quotations/train/76253 - Oliver Sebrantke zum dritten Mal Sieger beim Marathon in Bremen (2013-10-07).pretty.json\n", + "76253 - Oliver Sebrantke zum dritten Mal Sieger beim Marathon in Bremen (2013-10-07)\n", + "Processing datasets/german-quotations/train/45362 - Winterwelt am Potsdamer Platz ist eröffnet (2007-11-05).pretty.json\n", + "45362 - Winterwelt am Potsdamer Platz ist eröffnet (2007-11-05)\n", + "Processing datasets/german-quotations/train/68853 - Mordversuch in München: Eine Beziehung war 20-jähriger Schülerin nicht genug (2011-06-27).pretty.json\n", + "68853 - Mordversuch in München: Eine Beziehung war 20-jähriger Schülerin nicht genug (2011-06-27)\n", + "Processing datasets/german-quotations/train/39451 - Zehn Tote bei Brand in Moskauer Nachtclub (2007-03-26).pretty.json\n", + "39451 - Zehn Tote bei Brand in Moskauer Nachtclub (2007-03-26)\n", + "Processing datasets/german-quotations/train/53748 - Deutsche Big Brother Awards 2008 in Bielefeld verliehen (2008-10-27).pretty.json\n", + "53748 - Deutsche Big Brother Awards 2008 in Bielefeld verliehen (2008-10-27)\n", + "Processing datasets/german-quotations/train/3882 - Keine Dividende bei Beate Uhse (2005-03-11).pretty.json\n", + "3882 - Keine Dividende bei Beate Uhse (2005-03-11)\n", + "Processing datasets/german-quotations/train/62691 - Oderhochwasser erreicht Brandenburg (2010-05-26).pretty.json\n", + "62691 - Oderhochwasser erreicht Brandenburg (2010-05-26)\n", + "Processing datasets/german-quotations/train/25328 - Italien: Mafiaboss Bernardo Provenzano verhaftet (2006-04-12).pretty.json\n", + "25328 - Italien: Mafiaboss Bernardo Provenzano verhaftet (2006-04-12)\n", + "Processing datasets/german-quotations/train/2028 - Luftschiffhalle wird zum Urlaubsparadies (2004-12-19).pretty.json\n", + "2028 - Luftschiffhalle wird zum Urlaubsparadies (2004-12-19)\n", + "Processing datasets/german-quotations/train/6385 - Regierungsgebäude in Kirgisistan besetzt (2005-06-17).pretty.json\n", + "6385 - Regierungsgebäude in Kirgisistan besetzt (2005-06-17)\n", + "Processing datasets/german-quotations/train/54054 - BVG setzt Schutzfolien gegen Vandalismus ein (2008-11-16).pretty.json\n", + "54054 - BVG setzt Schutzfolien gegen Vandalismus ein (2008-11-16)\n", + "Processing datasets/german-quotations/train/26203 - Amyotrophische Lateralsklerose: Risiko kann halbiert werden (2006-05-04).pretty.json\n", + "26203 - Amyotrophische Lateralsklerose: Risiko kann halbiert werden (2006-05-04)\n", + "Processing datasets/german-quotations/train/68535 - Non liquet – Kachelmann freigesprochen (2011-06-04).pretty.json\n", + "68535 - Non liquet – Kachelmann freigesprochen (2011-06-04)\n", + "Processing datasets/german-quotations/train/52151 - Steigende Fluggastzahlen für Billigflieger Ryanair am Standort Bremen (2008-08-15).pretty.json\n", + "52151 - Steigende Fluggastzahlen für Billigflieger Ryanair am Standort Bremen (2008-08-15)\n", + "Processing datasets/german-quotations/train/28062 - Alan García gewinnt die Präsidentenwahl in Peru (2006-06-05).pretty.json\n", + "28062 - Alan García gewinnt die Präsidentenwahl in Peru (2006-06-05)\n", + "Processing datasets/german-quotations/train/13741 - Astronauten für „Shenzhou 6“ benannt - Start wird erstmals live übertragen (2005-10-11).pretty.json\n", + "13741 - Astronauten für „Shenzhou 6“ benannt - Start wird erstmals live übertragen (2005-10-11)\n", + "Processing datasets/german-quotations/train/77429 - Afghanistan: „Während der Wahl wurde von allen Seiten betrogen“ (2014-09-22).pretty.json\n", + "77429 - Afghanistan: „Während der Wahl wurde von allen Seiten betrogen“ (2014-09-22)\n", + "Processing datasets/german-quotations/train/12611 - Dritte Notlandung eines Flugzeuges in Italien innerhalb weniger Wochen (2005-09-18).pretty.json\n", + "12611 - Dritte Notlandung eines Flugzeuges in Italien innerhalb weniger Wochen (2005-09-18)\n", + "Processing datasets/german-quotations/train/35026 - Österreich: Zusammenstoß zwischen Hubschrauber und Kleinflugzeug (2006-11-15).pretty.json\n", + "35026 - Österreich: Zusammenstoß zwischen Hubschrauber und Kleinflugzeug (2006-11-15)\n", + "Processing datasets/german-quotations/train/82834 - Katastrophen-Warnungen bundesweit verfügbar (2019-02-07).pretty.json\n", + "82834 - Katastrophen-Warnungen bundesweit verfügbar (2019-02-07)\n", + "Processing datasets/german-quotations/train/71936 - Kony 2012 – Kritik an den Filmemachern (2012-03-11).pretty.json\n", + "71936 - Kony 2012 – Kritik an den Filmemachern (2012-03-11)\n", + "Processing datasets/german-quotations/train/66614 - SPD-regierte Bundesländer und Grüne bringen Laufzeitverlängerung für Atomkraftwerke vor das Bundesverfassungsgericht (2011-02-28).pretty.json\n", + "66614 - SPD-regierte Bundesländer und Grüne bringen Laufzeitverlängerung für Atomkraftwerke vor das Bundesverfassungsgericht (2011-02-28)\n", + "Processing datasets/german-quotations/train/23441 - Aus in der Champions League: FC Bayern München im Achtelfinale ausgeschieden (2006-03-09).pretty.json\n", + "23441 - Aus in der Champions League: FC Bayern München im Achtelfinale ausgeschieden (2006-03-09)\n", + "Processing datasets/german-quotations/train/79492 - B.Z. erklärt: Der typische Berliner Linksradikale wohnt noch daheim bei Mutti (2016-01-31).pretty.json\n", + "79492 - B.Z. erklärt: Der typische Berliner Linksradikale wohnt noch daheim bei Mutti (2016-01-31)\n", + "Processing datasets/german-quotations/train/72918 - Japan: 64-jähriger Boss einer Yakuza-Bande kommt gegen 15 Millionen Euro Kaution frei (2012-06-12).pretty.json\n", + "72918 - Japan: 64-jähriger Boss einer Yakuza-Bande kommt gegen 15 Millionen Euro Kaution frei (2012-06-12)\n", + "Processing datasets/german-quotations/train/61190 - Afghanistan: Auch zivile Opfer bei Militäroffensive in der Provinz Helmand (2010-02-16).pretty.json\n", + "61190 - Afghanistan: Auch zivile Opfer bei Militäroffensive in der Provinz Helmand (2010-02-16)\n", + "Processing datasets/german-quotations/train/43924 - Unerwartete Wende im Lokführer-Tarifstreit (2007-08-30).pretty.json\n", + "43924 - Unerwartete Wende im Lokführer-Tarifstreit (2007-08-30)\n", + "Processing datasets/german-quotations/train/66791 - Kairo: Tote und Verletzte bei Kämpfen zwischen Muslimen und koptischen Christen (2011-03-09).pretty.json\n", + "66791 - Kairo: Tote und Verletzte bei Kämpfen zwischen Muslimen und koptischen Christen (2011-03-09)\n", + "Processing datasets/german-quotations/train/70712 - München: Drei Polizeifahrzeuge gehen in Flammen auf (2011-12-08).pretty.json\n", + "70712 - München: Drei Polizeifahrzeuge gehen in Flammen auf (2011-12-08)\n", + "Processing datasets/german-quotations/train/69157 - Abschiedsmission der Raumfähre Atlantis beendet (2011-07-23).pretty.json\n", + "69157 - Abschiedsmission der Raumfähre Atlantis beendet (2011-07-23)\n", + "Processing datasets/german-quotations/train/43595 - Sechsfacher Mord in Duisburg – Mafia-Hintergrund vermutet (2007-08-15).pretty.json\n", + "43595 - Sechsfacher Mord in Duisburg – Mafia-Hintergrund vermutet (2007-08-15)\n", + "Processing datasets/german-quotations/train/21319 - „Deus caritas est“ in dritter Auflage erschienen (2006-02-02).pretty.json\n", + "21319 - „Deus caritas est“ in dritter Auflage erschienen (2006-02-02)\n", + "Processing datasets/german-quotations/train/80888 - Gibt es Frieden zu Ostern in Syrien? (2017-04-11).pretty.json\n", + "80888 - Gibt es Frieden zu Ostern in Syrien? (2017-04-11)\n", + "Processing datasets/german-quotations/train/12953 - Hans Küng zu Besuch bei Papst Benedikt XVI. (2005-09-26).pretty.json\n", + "12953 - Hans Küng zu Besuch bei Papst Benedikt XVI. (2005-09-26)\n", + "Processing datasets/german-quotations/train/47872 - Neunzehnter Spieltag der Fußball-Bundesliga 2007-08 (2008-02-11).pretty.json\n", + "47872 - Neunzehnter Spieltag der Fußball-Bundesliga 2007-08 (2008-02-11)\n", + "Processing datasets/german-quotations/train/72414 - Oberpfalz: Polizei beendet familiäre Geiselnahme unblutig (2012-04-16).pretty.json\n", + "72414 - Oberpfalz: Polizei beendet familiäre Geiselnahme unblutig (2012-04-16)\n", + "Processing datasets/german-quotations/train/53890 - Über 120 Tote durch Hochwasser in Vietnam und China (2008-11-05).pretty.json\n", + "53890 - Über 120 Tote durch Hochwasser in Vietnam und China (2008-11-05)\n", + "Processing datasets/german-quotations/train/64897 - Parlamentswahl in Moldawien: weiterhin keine klaren Mehrheiten (2010-11-30).pretty.json\n", + "64897 - Parlamentswahl in Moldawien: weiterhin keine klaren Mehrheiten (2010-11-30)\n", + "Processing datasets/german-quotations/train/45940 - Deutscher Wetterdienst gab für gestern eine Wind--Sturmwarnung für das gesamte Bundesgebiet heraus (2007-12-03).pretty.json\n", + "45940 - Deutscher Wetterdienst gab für gestern eine Wind--Sturmwarnung für das gesamte Bundesgebiet heraus (2007-12-03)\n", + "Processing datasets/german-quotations/train/1212 - IAEA will iranische Militäranlagen nicht untersuchen (2004-12-04).pretty.json\n", + "1212 - IAEA will iranische Militäranlagen nicht untersuchen (2004-12-04)\n", + "Processing datasets/german-quotations/train/50415 - Ehemaliger Vizepräsident der Demokratischen Republik Kongo, Jean-Pierre Bemba, in Belgien verhaftet (2008-05-25).pretty.json\n", + "50415 - Ehemaliger Vizepräsident der Demokratischen Republik Kongo, Jean-Pierre Bemba, in Belgien verhaftet (2008-05-25)\n", + "Processing datasets/german-quotations/train/81799 - Korruptionsverdacht bei rheinland-pfälzischer Straßenbaubehörde, Behördenleiter begeht Suizid (2017-10-20).pretty.json\n", + "81799 - Korruptionsverdacht bei rheinland-pfälzischer Straßenbaubehörde, Behördenleiter begeht Suizid (2017-10-20)\n", + "Processing datasets/german-quotations/train/83783 - Handball-EM: Deutschland gewinnt gegen Weißrussland, Österreich unterliegt Kroatien (2020-01-16).pretty.json\n", + "83783 - Handball-EM: Deutschland gewinnt gegen Weißrussland, Österreich unterliegt Kroatien (2020-01-16)\n", + "Processing datasets/german-quotations/train/53754 - Großbrand zerstört Möbellager in Hermeskeil (2008-10-27).pretty.json\n", + "53754 - Großbrand zerstört Möbellager in Hermeskeil (2008-10-27)\n", + "Processing datasets/german-quotations/train/18645 - Verhandlungen über die Freilassung von Jürgen Chrobog und seiner Familie (2005-12-29).pretty.json\n", + "18645 - Verhandlungen über die Freilassung von Jürgen Chrobog und seiner Familie (2005-12-29)\n", + "Processing datasets/german-quotations/train/59510 - Schläfer des Kalten Krieges in Bremen entdeckt? (2009-10-29).pretty.json\n", + "59510 - Schläfer des Kalten Krieges in Bremen entdeckt? (2009-10-29)\n", + "Processing datasets/german-quotations/train/5178 - Der Papst-Golf schlägt alle Rekorde (2005-05-05).pretty.json\n", + "5178 - Der Papst-Golf schlägt alle Rekorde (2005-05-05)\n", + "Processing datasets/german-quotations/train/64872 - Wulff: Deutschland trägt Verantwortung für das Existenzrecht Israels (2010-11-28).pretty.json\n", + "64872 - Wulff: Deutschland trägt Verantwortung für das Existenzrecht Israels (2010-11-28)\n", + "Processing datasets/german-quotations/train/24349 - CDU-Spitzenkandidat Christoph Böhr von allen Ämtern zurückgetreten (2006-03-27).pretty.json\n", + "24349 - CDU-Spitzenkandidat Christoph Böhr von allen Ämtern zurückgetreten (2006-03-27)\n", + "Processing datasets/german-quotations/train/59262 - Herta Müller erhält den Nobelpreis für Literatur (2009-10-08).pretty.json\n", + "59262 - Herta Müller erhält den Nobelpreis für Literatur (2009-10-08)\n", + "Processing datasets/german-quotations/train/61951 - Frachter am Great Barrrier Reef droht auseinanderzubrechen (2010-04-06).pretty.json\n", + "61951 - Frachter am Great Barrrier Reef droht auseinanderzubrechen (2010-04-06)\n", + "Processing datasets/german-quotations/train/71599 - Nina Hagen und Tochter Cosma Shiva ziehen aufs Land (2012-02-13).pretty.json\n", + "71599 - Nina Hagen und Tochter Cosma Shiva ziehen aufs Land (2012-02-13)\n", + "Processing datasets/german-quotations/train/71772 - Serbien wird den Status eines EU-Beitrittskandidaten erhalten (2012-02-25).pretty.json\n", + "71772 - Serbien wird den Status eines EU-Beitrittskandidaten erhalten (2012-02-25)\n", + "Processing datasets/german-quotations/train/35769 - Schneesturm im Mittleren Westen der USA (2006-12-03).pretty.json\n", + "35769 - Schneesturm im Mittleren Westen der USA (2006-12-03)\n", + "Processing datasets/german-quotations/train/14879 - Erstmals Zapfenstreich zum Bundeswehr-Jubiläum vor dem Berliner Reichstag (2005-10-27).pretty.json\n", + "14879 - Erstmals Zapfenstreich zum Bundeswehr-Jubiläum vor dem Berliner Reichstag (2005-10-27)\n", + "Processing datasets/german-quotations/train/66793 - Gouverneur des US-Bundesstaates Illinois unterzeichnet Gesetz zur Abschaffung der Todesstrafe (2011-03-09).pretty.json\n", + "66793 - Gouverneur des US-Bundesstaates Illinois unterzeichnet Gesetz zur Abschaffung der Todesstrafe (2011-03-09)\n", + "Processing datasets/german-quotations/train/37033 - Nach 25 Jahren Anklage wegen Sexualmordes an 16-Jähriger (2007-01-11).pretty.json\n", + "37033 - Nach 25 Jahren Anklage wegen Sexualmordes an 16-Jähriger (2007-01-11)\n", + "Processing datasets/german-quotations/train/19560 - Flugverbot für peruanische Fluggesellschaft TANS (2006-01-09).pretty.json\n", + "19560 - Flugverbot für peruanische Fluggesellschaft TANS (2006-01-09)\n", + "Processing datasets/german-quotations/train/58343 - Porsche-Übernahme durch die Volkswagen AG perfekt – Wiedeking tritt zurück (2009-07-24).pretty.json\n", + "58343 - Porsche-Übernahme durch die Volkswagen AG perfekt – Wiedeking tritt zurück (2009-07-24)\n", + "Processing datasets/german-quotations/train/72767 - Erdbeben in Norditalien: sechs Tote und erhebliche Schäden an Kulturdenkmälern (2012-05-21).pretty.json\n", + "72767 - Erdbeben in Norditalien: sechs Tote und erhebliche Schäden an Kulturdenkmälern (2012-05-21)\n", + "Processing datasets/german-quotations/train/83589 - Super Bowl LIV: Halbzeitshow mit Jennifer Lopez und Shakira (2019-09-29).pretty.json\n", + "83589 - Super Bowl LIV: Halbzeitshow mit Jennifer Lopez und Shakira (2019-09-29)\n", + "Processing datasets/german-quotations/train/24558 - Verfahren wegen Körperverletzung gegen Formel-1-Pilot Christian Klien eingestellt (2006-03-31).pretty.json\n", + "24558 - Verfahren wegen Körperverletzung gegen Formel-1-Pilot Christian Klien eingestellt (2006-03-31)\n", + "Processing datasets/german-quotations/train/32740 - Der DFB hat einen neuen Vorsitzenden: Theo Zwanziger (2006-09-08).pretty.json\n", + "32740 - Der DFB hat einen neuen Vorsitzenden: Theo Zwanziger (2006-09-08)\n", + "Processing datasets/german-quotations/train/44436 - LKW-Brand: Sauerlandlinie stundenlang blockiert (2007-09-25).pretty.json\n", + "44436 - LKW-Brand: Sauerlandlinie stundenlang blockiert (2007-09-25)\n", + "Processing datasets/german-quotations/train/19835 - Streit um Montezumas Kopfschmuck (2006-01-11).pretty.json\n", + "19835 - Streit um Montezumas Kopfschmuck (2006-01-11)\n", + "Processing datasets/german-quotations/train/33591 - Milliardär Friedrich Karl Flick gestorben (2006-10-06).pretty.json\n", + "33591 - Milliardär Friedrich Karl Flick gestorben (2006-10-06)\n", + "Processing datasets/german-quotations/train/15588 - Mutmaßlicher Entführer der Göttinger Familie Wallert verhaftet (2005-11-06).pretty.json\n", + "15588 - Mutmaßlicher Entführer der Göttinger Familie Wallert verhaftet (2005-11-06)\n", + "Processing datasets/german-quotations/train/15010 - Mord an junger Frau in Darmstadt (2005-10-28).pretty.json\n", + "15010 - Mord an junger Frau in Darmstadt (2005-10-28)\n", + "Processing datasets/german-quotations/train/69852 - Anklage fordert 7 Jahre Haft für Timoschenko (2011-09-27).pretty.json\n", + "69852 - Anklage fordert 7 Jahre Haft für Timoschenko (2011-09-27)\n", + "Processing datasets/german-quotations/train/5310 - Weiterhin Probleme mit ALG-II-Software (2005-05-11).pretty.json\n", + "5310 - Weiterhin Probleme mit ALG-II-Software (2005-05-11)\n", + "Processing datasets/german-quotations/train/54805 - EinsFestival HD mit Weihnachtsshowcase 2008 gestartet (2008-12-23).pretty.json\n", + "54805 - EinsFestival HD mit Weihnachtsshowcase 2008 gestartet (2008-12-23)\n", + "Processing datasets/german-quotations/train/82348 - Gemeinschaftsschulen in Baden-Württemberg zu erfolgreich für manche? (2018-03-20).pretty.json\n", + "82348 - Gemeinschaftsschulen in Baden-Württemberg zu erfolgreich für manche? (2018-03-20)\n", + "Processing datasets/german-quotations/train/25147 - Bundestag genehmigte Privatisierung der „Deutschen Flugsicherung“ (DFS) (2006-04-09).pretty.json\n", + "25147 - Bundestag genehmigte Privatisierung der „Deutschen Flugsicherung“ (DFS) (2006-04-09)\n", + "Processing datasets/german-quotations/train/5882 - „Verlorene Hörspielmusik“ erscheint nach 20 Jahren auf CD (2005-06-04).pretty.json\n", + "5882 - „Verlorene Hörspielmusik“ erscheint nach 20 Jahren auf CD (2005-06-04)\n", + "Processing datasets/german-quotations/train/81848 - Sizilien: Mafia Boss befiehlt die Ermordung der eigenen Tochter (2017-11-05).pretty.json\n", + "81848 - Sizilien: Mafia Boss befiehlt die Ermordung der eigenen Tochter (2017-11-05)\n", + "Processing datasets/german-quotations/train/39672 - Neuigkeiten zum Actionspiel „Grand Theft Auto IV“ (2007-03-31).pretty.json\n", + "39672 - Neuigkeiten zum Actionspiel „Grand Theft Auto IV“ (2007-03-31)\n", + "Processing datasets/german-quotations/train/48012 - Deutscher in Somalia entführt und wieder befreit (2008-02-13).pretty.json\n", + "48012 - Deutscher in Somalia entführt und wieder befreit (2008-02-13)\n", + "Processing datasets/german-quotations/train/19927 - Eine Maus wurde zur Brandstifterin (2006-01-13).pretty.json\n", + "19927 - Eine Maus wurde zur Brandstifterin (2006-01-13)\n", + "Processing datasets/german-quotations/train/29373 - Argentiniens Nationaltrainer José Pekerman zurückgetreten (2006-07-01).pretty.json\n", + "29373 - Argentiniens Nationaltrainer José Pekerman zurückgetreten (2006-07-01)\n", + "Processing datasets/german-quotations/train/24259 - Deutschlands Fußball-WM-Gruppengegner Ecuador in Nöten (2006-03-25).pretty.json\n", + "24259 - Deutschlands Fußball-WM-Gruppengegner Ecuador in Nöten (2006-03-25)\n", + "Processing datasets/german-quotations/train/25900 - Ground Zero: Neubau des World Trade Center begonnen (2006-04-28).pretty.json\n", + "25900 - Ground Zero: Neubau des World Trade Center begonnen (2006-04-28)\n", + "Processing datasets/german-quotations/train/25582 - Documenta 12 mit neuem grafischen Erscheinungsbild (2006-04-20).pretty.json\n", + "25582 - Documenta 12 mit neuem grafischen Erscheinungsbild (2006-04-20)\n", + "Processing datasets/german-quotations/train/17869 - John F. Kennedys Armbanduhr für 350.000 US-Dollar versteigert (2005-12-20).pretty.json\n", + "17869 - John F. Kennedys Armbanduhr für 350.000 US-Dollar versteigert (2005-12-20)\n", + "Processing datasets/german-quotations/train/63711 - Koalition einigt sich auf Laufzeitverlängerung für Atomkraftwerke (2010-09-07).pretty.json\n", + "63711 - Koalition einigt sich auf Laufzeitverlängerung für Atomkraftwerke (2010-09-07)\n", + "Processing datasets/german-quotations/train/25851 - Moritz Schenk Graf von Stauffenberg: 1,1 Promille Alkohol im Blut (2006-04-27).pretty.json\n", + "25851 - Moritz Schenk Graf von Stauffenberg: 1,1 Promille Alkohol im Blut (2006-04-27)\n", + "Processing datasets/german-quotations/train/29108 - Ex-Präsident von Liberia nach Den Haag überstellt (2006-06-27).pretty.json\n", + "29108 - Ex-Präsident von Liberia nach Den Haag überstellt (2006-06-27)\n", + "Processing datasets/german-quotations/train/65500 - Frankreich: Kommunisten verschieben Entscheidung über Präsidentschaftskandidatur (2011-01-10).pretty.json\n", + "65500 - Frankreich: Kommunisten verschieben Entscheidung über Präsidentschaftskandidatur (2011-01-10)\n", + "Processing datasets/german-quotations/train/24911 - Nach Parlamentswahl in Kanada: Stephen Harper stellte Regierungsprogramm vor (2006-04-05).pretty.json\n", + "24911 - Nach Parlamentswahl in Kanada: Stephen Harper stellte Regierungsprogramm vor (2006-04-05)\n", + "Processing datasets/german-quotations/train/34731 - Putschvorbereitungen in Fidschi (2006-11-04).pretty.json\n", + "34731 - Putschvorbereitungen in Fidschi (2006-11-04)\n", + "Processing datasets/german-quotations/train/45812 - Kaiserslautern: Tödlicher Verkehrsunfall auf der BAB 6 (2007-11-25).pretty.json\n", + "45812 - Kaiserslautern: Tödlicher Verkehrsunfall auf der BAB 6 (2007-11-25)\n", + "Processing datasets/german-quotations/train/6399 - Giovanni Trapattoni wird neuer Trainer beim VFB Stuttgart (2005-06-17).pretty.json\n", + "6399 - Giovanni Trapattoni wird neuer Trainer beim VFB Stuttgart (2005-06-17)\n", + "Processing datasets/german-quotations/train/12599 - Fall des eBay-Babys doch anders als gedacht (2005-09-17).pretty.json\n", + "12599 - Fall des eBay-Babys doch anders als gedacht (2005-09-17)\n", + "Processing datasets/german-quotations/train/58397 - Bombenexplosion auf der Baleareninsel Mallorca (2009-08-02).pretty.json\n", + "58397 - Bombenexplosion auf der Baleareninsel Mallorca (2009-08-02)\n", + "Processing datasets/german-quotations/train/18208 - Hubschrauber mit drei Besatzungsmitgliedern in Sibirien vermisst (2005-12-24).pretty.json\n", + "18208 - Hubschrauber mit drei Besatzungsmitgliedern in Sibirien vermisst (2005-12-24)\n", + "Processing datasets/german-quotations/train/18718 - 13 Tonnen Kerosin über der Schweiz abgelassen (2005-12-31).pretty.json\n", + "18718 - 13 Tonnen Kerosin über der Schweiz abgelassen (2005-12-31)\n", + "Processing datasets/german-quotations/train/34485 - November 2006: Treffen der großen Schachspieler in Moskau (2006-10-30).pretty.json\n", + "34485 - November 2006: Treffen der großen Schachspieler in Moskau (2006-10-30)\n", + "Processing datasets/german-quotations/train/65962 - München: Italienische Touristin begeht räuberischen Diebstahl im Hauptbahnhof (2011-02-02).pretty.json\n", + "65962 - München: Italienische Touristin begeht räuberischen Diebstahl im Hauptbahnhof (2011-02-02)\n", + "Processing datasets/german-quotations/train/37751 - Handball-Weltmeisterschaft: Deutschland im Finale (2007-02-02).pretty.json\n", + "37751 - Handball-Weltmeisterschaft: Deutschland im Finale (2007-02-02)\n", + "Processing datasets/german-quotations/train/17843 - Fred Delmare leidet an Alzheimer (2005-12-18).pretty.json\n", + "17843 - Fred Delmare leidet an Alzheimer (2005-12-18)\n", + "Processing datasets/german-quotations/train/13380 - Elf Verletzte beim Transport von Fallschirmspringern im US-amerikanischen DeLand (2005-10-04).pretty.json\n", + "13380 - Elf Verletzte beim Transport von Fallschirmspringern im US-amerikanischen DeLand (2005-10-04)\n", + "Processing datasets/german-quotations/train/5167 - Trigema-Chef Grupp steht hinter Münteferings Kapitalismus-Kritik (2005-05-11).pretty.json\n", + "5167 - Trigema-Chef Grupp steht hinter Münteferings Kapitalismus-Kritik (2005-05-11)\n", + "Processing datasets/german-quotations/train/74636 - Deutsche Bischofskonferenz stoppt kriminologische Studie zum sexuellen Missbrauch (2013-01-13).pretty.json\n", + "74636 - Deutsche Bischofskonferenz stoppt kriminologische Studie zum sexuellen Missbrauch (2013-01-13)\n", + "Processing datasets/german-quotations/train/42185 - VW-Affäre: SPD-Landtagsabgeordneter Lenz legt Mandat nieder (2007-06-15).pretty.json\n", + "42185 - VW-Affäre: SPD-Landtagsabgeordneter Lenz legt Mandat nieder (2007-06-15)\n", + "Processing datasets/german-quotations/train/38098 - Angela Merkel fordert mehr benzinsparende Autos (2007-02-12).pretty.json\n", + "38098 - Angela Merkel fordert mehr benzinsparende Autos (2007-02-12)\n", + "Processing datasets/german-quotations/train/27312 - Verfassungsschutzbericht 2005 vorgelegt – Anstieg rechter Gewalt (2006-05-24).pretty.json\n", + "27312 - Verfassungsschutzbericht 2005 vorgelegt – Anstieg rechter Gewalt (2006-05-24)\n", + "Processing datasets/german-quotations/train/60160 - Bürgerkriegsähnliche Zustände in Teheran (2009-12-27).pretty.json\n", + "60160 - Bürgerkriegsähnliche Zustände in Teheran (2009-12-27)\n", + "Processing datasets/german-quotations/train/54964 - Jahresbilanz 2008 von „Reporter ohne Grenzen“: Journalisten leben weiter gefährlich (2008-12-30).pretty.json\n", + "54964 - Jahresbilanz 2008 von „Reporter ohne Grenzen“: Journalisten leben weiter gefährlich (2008-12-30)\n", + "Processing datasets/german-quotations/train/20136 - ZDF DokuDrama: Karol Wojtyla – Geheimnisse eines Papstes (2006-01-16).pretty.json\n", + "20136 - ZDF DokuDrama: Karol Wojtyla – Geheimnisse eines Papstes (2006-01-16)\n", + "Processing datasets/german-quotations/train/14074 - Sabotage mit Metallgegenstand in einem Schwelmer Maisfeld (2005-10-21).pretty.json\n", + "14074 - Sabotage mit Metallgegenstand in einem Schwelmer Maisfeld (2005-10-21)\n", + "Processing datasets/german-quotations/train/50978 - Afghanistans Präsident Karzai will „Kampf gegen den Terror“ auf das pakistanische Grenzgebiet ausweiten (2008-06-15).pretty.json\n", + "50978 - Afghanistans Präsident Karzai will „Kampf gegen den Terror“ auf das pakistanische Grenzgebiet ausweiten (2008-06-15)\n", + "Processing datasets/german-quotations/train/69903 - Arktis: Forscher von Ozonloch beunruhigt (2011-10-04).pretty.json\n", + "69903 - Arktis: Forscher von Ozonloch beunruhigt (2011-10-04)\n", + "Processing datasets/german-quotations/train/68514 - Warnung vor spanischen Gurken ‒ Zapatero verlangt Entschädigung von Deutschland (2011-06-02).pretty.json\n", + "68514 - Warnung vor spanischen Gurken ‒ Zapatero verlangt Entschädigung von Deutschland (2011-06-02)\n", + "Processing datasets/german-quotations/train/48238 - Pétanque: Deutsches Damenteam Dritte in Genf (2008-02-24).pretty.json\n", + "48238 - Pétanque: Deutsches Damenteam Dritte in Genf (2008-02-24)\n", + "Processing datasets/german-quotations/train/8561 - Zukunftsvertrag regelt die Finanzierung der niedersächsischen Hochschulen (2005-07-13).pretty.json\n", + "8561 - Zukunftsvertrag regelt die Finanzierung der niedersächsischen Hochschulen (2005-07-13)\n", + "Processing datasets/german-quotations/train/23893 - Abschlussbericht zum Flugzeugabsturz in Athen fertiggestellt (2006-03-19).pretty.json\n", + "23893 - Abschlussbericht zum Flugzeugabsturz in Athen fertiggestellt (2006-03-19)\n", + "Processing datasets/german-quotations/train/29688 - Fußballskandal Italien: Weit reichende Strafen von Untersuchungsrichter gefordert (2006-07-07).pretty.json\n", + "29688 - Fußballskandal Italien: Weit reichende Strafen von Untersuchungsrichter gefordert (2006-07-07)\n", + "Processing datasets/german-quotations/train/45701 - ATTAC: Streit um Unterstützung des Lokführerstreiks (2007-11-18).pretty.json\n", + "45701 - ATTAC: Streit um Unterstützung des Lokführerstreiks (2007-11-18)\n", + "Processing datasets/german-quotations/train/28728 - US-Soldat wegen Tötung des italienischen Geheimdienstagenten Calipari angeklagt (2006-06-22).pretty.json\n", + "28728 - US-Soldat wegen Tötung des italienischen Geheimdienstagenten Calipari angeklagt (2006-06-22)\n", + "Processing datasets/german-quotations/train/56998 - Steigende Spannungen in Thailand durch Proteste von Regierungsgegnern (2009-04-07).pretty.json\n", + "56998 - Steigende Spannungen in Thailand durch Proteste von Regierungsgegnern (2009-04-07)\n", + "Processing datasets/german-quotations/train/27100 - Nationalspieler Philipp Lahm reist am Samstag nach Sardinien (2006-05-19).pretty.json\n", + "27100 - Nationalspieler Philipp Lahm reist am Samstag nach Sardinien (2006-05-19)\n", + "Processing datasets/german-quotations/train/41379 - NBA Playoffs 2007: Halbfinale sind komplett (2007-05-19).pretty.json\n", + "41379 - NBA Playoffs 2007: Halbfinale sind komplett (2007-05-19)\n", + "Processing datasets/german-quotations/train/17865 - Der vor Genua vermisste Pilot ist möglicherweise noch am Leben (2005-12-19).pretty.json\n", + "17865 - Der vor Genua vermisste Pilot ist möglicherweise noch am Leben (2005-12-19)\n", + "Processing datasets/german-quotations/train/24321 - Nach dem Absturz der Trägerrakete „Falcon 1“: Elon Musk will Ursachenforschung betreiben (2006-03-27).pretty.json\n", + "24321 - Nach dem Absturz der Trägerrakete „Falcon 1“: Elon Musk will Ursachenforschung betreiben (2006-03-27)\n", + "Processing datasets/german-quotations/train/65110 - Guttenberg besucht Truppen in Afghanistan (2010-12-14).pretty.json\n", + "65110 - Guttenberg besucht Truppen in Afghanistan (2010-12-14)\n", + "Processing datasets/german-quotations/train/13587 - Notlandung eines polnischen Löschhubschraubers auf einem Golfplatz im Walzbachtal (2005-10-09).pretty.json\n", + "13587 - Notlandung eines polnischen Löschhubschraubers auf einem Golfplatz im Walzbachtal (2005-10-09)\n", + "Processing datasets/german-quotations/train/54334 - Berlin: Rot-rote Koalition plant Einführung einer „Schülerdatei“ (2008-11-26).pretty.json\n", + "54334 - Berlin: Rot-rote Koalition plant Einführung einer „Schülerdatei“ (2008-11-26)\n", + "Processing datasets/german-quotations/train/27206 - Thaksin Shinawatra kündigt Rückkehr in die Politik an (2006-05-21).pretty.json\n", + "27206 - Thaksin Shinawatra kündigt Rückkehr in die Politik an (2006-05-21)\n", + "Processing datasets/german-quotations/train/45074 - Prozessauftakt im Fall „Kevin“: Angeklagter Ziehvater schweigt (2007-10-24).pretty.json\n", + "45074 - Prozessauftakt im Fall „Kevin“: Angeklagter Ziehvater schweigt (2007-10-24)\n", + "Processing datasets/german-quotations/train/80950 - Mord an Rentner in Cleveland: Video der Tat auf Facebook (2017-04-17).pretty.json\n", + "80950 - Mord an Rentner in Cleveland: Video der Tat auf Facebook (2017-04-17)\n", + "Processing datasets/german-quotations/train/65490 - US-Gericht fordert Twitter zur Herausgabe von personenbezogenen Daten über WikiLeaks-Kontaktpersonen auf (2011-01-08).pretty.json\n", + "65490 - US-Gericht fordert Twitter zur Herausgabe von personenbezogenen Daten über WikiLeaks-Kontaktpersonen auf (2011-01-08)\n", + "Processing datasets/german-quotations/train/69135 - Lebensmittelklarheit.de geht online und dann in die Knie (2011-07-20).pretty.json\n", + "69135 - Lebensmittelklarheit.de geht online und dann in die Knie (2011-07-20)\n", + "Processing datasets/german-quotations/train/82489 - Schulen in Deutschland: Sind die Grenzen der Inklusion erreicht? (2018-07-08).pretty.json\n", + "82489 - Schulen in Deutschland: Sind die Grenzen der Inklusion erreicht? (2018-07-08)\n", + "Processing datasets/german-quotations/train/64928 - Deutschland: Innenminister de Maizière legt Gesetzentwurf für verbesserten Datenschutz im Internet vor (2010-12-01).pretty.json\n", + "64928 - Deutschland: Innenminister de Maizière legt Gesetzentwurf für verbesserten Datenschutz im Internet vor (2010-12-01)\n", + "Processing datasets/german-quotations/train/25051 - Iserlohn: Mutmaßlicher Täter im Doppelmordfall war einer der drei verhafteten Männer (2006-04-07).pretty.json\n", + "25051 - Iserlohn: Mutmaßlicher Täter im Doppelmordfall war einer der drei verhafteten Männer (2006-04-07)\n", + "Processing datasets/german-quotations/train/3659 - Dschungelbuch-Premiere in Nürnberg (2005-03-02).pretty.json\n", + "3659 - Dschungelbuch-Premiere in Nürnberg (2005-03-02)\n", + "Processing datasets/german-quotations/train/51123 - Barack Obama will auf staatliche Zuschüsse für seinen Wahlkampf verzichten (2008-06-21).pretty.json\n", + "51123 - Barack Obama will auf staatliche Zuschüsse für seinen Wahlkampf verzichten (2008-06-21)\n", + "Processing datasets/german-quotations/train/43106 - Sperre der A9 wegen Installation einer Fußgängerbrücke (2007-07-29).pretty.json\n", + "43106 - Sperre der A9 wegen Installation einer Fußgängerbrücke (2007-07-29)\n", + "Processing datasets/german-quotations/train/29022 - Hauptkamera des Hubble-Weltraumteleskops ausgefallen (2006-06-26).pretty.json\n", + "29022 - Hauptkamera des Hubble-Weltraumteleskops ausgefallen (2006-06-26)\n", + "Processing datasets/german-quotations/train/29721 - Großbrand in einem Elektrofeinmechanik-Betrieb in Olpe (2006-07-06).pretty.json\n", + "29721 - Großbrand in einem Elektrofeinmechanik-Betrieb in Olpe (2006-07-06)\n", + "Processing datasets/german-quotations/train/4555 - Macromedia-Entwicklerkonferenz: Neuer Flash-Player kommt noch dieses Jahr (2005-04-09).pretty.json\n", + "4555 - Macromedia-Entwicklerkonferenz: Neuer Flash-Player kommt noch dieses Jahr (2005-04-09)\n", + "Processing datasets/german-quotations/train/3354 - Hoyzer belastet erneut Schiedsrichterkollegen (2005-02-19).pretty.json\n", + "3354 - Hoyzer belastet erneut Schiedsrichterkollegen (2005-02-19)\n", + "Processing datasets/german-quotations/train/47258 - Oberhaupt der griechisch-orthodoxen Kirche verstorben (2008-01-28).pretty.json\n", + "47258 - Oberhaupt der griechisch-orthodoxen Kirche verstorben (2008-01-28)\n", + "Processing datasets/german-quotations/train/4389 - WM 2006 nach England verlegt (2005-04-01).pretty.json\n", + "4389 - WM 2006 nach England verlegt (2005-04-01)\n", + "Processing datasets/german-quotations/train/32521 - Kaiserslautern: Fahndung nach Grabscher (2006-09-03).pretty.json\n", + "32521 - Kaiserslautern: Fahndung nach Grabscher (2006-09-03)\n", + "Processing datasets/german-quotations/train/5156 - Drucksparten von Springer und Bertelsmann fusionieren (2005-05-03).pretty.json\n", + "5156 - Drucksparten von Springer und Bertelsmann fusionieren (2005-05-03)\n", + "Processing datasets/german-quotations/train/15720 - Unfall im Maisfeld: Jäger schießt versehentlich auf Mähdrescherfahrer (2005-11-11).pretty.json\n", + "15720 - Unfall im Maisfeld: Jäger schießt versehentlich auf Mähdrescherfahrer (2005-11-11)\n", + "Processing datasets/german-quotations/train/63732 - Tausende demonstrieren in Berlin für Freiheit statt Angst (2010-09-13).pretty.json\n", + "63732 - Tausende demonstrieren in Berlin für Freiheit statt Angst (2010-09-13)\n", + "Processing datasets/german-quotations/train/55767 - Schnellstes Elektroauto der Welt in Planung (2009-02-05).pretty.json\n", + "55767 - Schnellstes Elektroauto der Welt in Planung (2009-02-05)\n", + "Processing datasets/german-quotations/train/33398 - Delmenhorst: Gegner der rechten Kaderschmiede erneut als „Volksverräter“ beschimpft (2006-10-01).pretty.json\n", + "33398 - Delmenhorst: Gegner der rechten Kaderschmiede erneut als „Volksverräter“ beschimpft (2006-10-01)\n", + "Processing datasets/german-quotations/train/30797 - Chiang Mai erneut von schweren Überflutungen heimgesucht (2006-08-02).pretty.json\n", + "30797 - Chiang Mai erneut von schweren Überflutungen heimgesucht (2006-08-02)\n", + "Processing datasets/german-quotations/train/46026 - Kinder in Panama sollen in Zukunft Chinesisch lernen (2007-12-07).pretty.json\n", + "46026 - Kinder in Panama sollen in Zukunft Chinesisch lernen (2007-12-07)\n", + "Processing datasets/german-quotations/train/13246 - Krisentreffen der EU-Außenminister wegen Türkeibeitrittsverhandlungen (2005-10-02).pretty.json\n", + "13246 - Krisentreffen der EU-Außenminister wegen Türkeibeitrittsverhandlungen (2005-10-02)\n", + "Processing datasets/german-quotations/train/41206 - Borussia Dortmund siegt gegen Schalke 04 (2007-05-12).pretty.json\n", + "41206 - Borussia Dortmund siegt gegen Schalke 04 (2007-05-12)\n", + "Processing datasets/german-quotations/train/84621 - Impfziel: Mehr als 80 Prozent der doppelt Geimpften auch geboostert (2022-01-06).pretty.json\n", + "84621 - Impfziel: Mehr als 80 Prozent der doppelt Geimpften auch geboostert (2022-01-06)\n", + "Processing datasets/german-quotations/train/59120 - Amerikanische Bibliothekenverbindung feiert „Woche des verbotenen Buches“ (2009-10-01).pretty.json\n", + "59120 - Amerikanische Bibliothekenverbindung feiert „Woche des verbotenen Buches“ (2009-10-01)\n", + "Processing datasets/german-quotations/train/28446 - Finnische Hunde jagen Bären (2006-06-14).pretty.json\n", + "28446 - Finnische Hunde jagen Bären (2006-06-14)\n", + "Processing datasets/german-quotations/train/12830 - Schwerstverletzte Kletterin mit Hubschrauber gerettet (2005-09-23).pretty.json\n", + "12830 - Schwerstverletzte Kletterin mit Hubschrauber gerettet (2005-09-23)\n", + "Processing datasets/german-quotations/train/59985 - Jüdische Siedler verüben Brandanschlag auf Moschee (2009-12-14).pretty.json\n", + "59985 - Jüdische Siedler verüben Brandanschlag auf Moschee (2009-12-14)\n", + "Processing datasets/german-quotations/train/10022 - Der Astronaut Neil Armstrong wird heute 75 Jahre alt (2005-08-05).pretty.json\n", + "10022 - Der Astronaut Neil Armstrong wird heute 75 Jahre alt (2005-08-05)\n", + "Processing datasets/german-quotations/train/22774 - Nach dem Oscar nun der César (2006-02-26).pretty.json\n", + "22774 - Nach dem Oscar nun der César (2006-02-26)\n", + "Processing datasets/german-quotations/train/36350 - Die Polizei: Dein Freund und Helfer (2006-12-19).pretty.json\n", + "36350 - Die Polizei: Dein Freund und Helfer (2006-12-19)\n", + "Processing datasets/german-quotations/train/81262 - Ranking: Wertvollste Konzerne der Welt kommen aus den USA – China holt auf (2017-07-01).pretty.json\n", + "81262 - Ranking: Wertvollste Konzerne der Welt kommen aus den USA – China holt auf (2017-07-01)\n", + "Processing datasets/german-quotations/train/35999 - HSV-Sieg durch „Stinkefinger-Affäre“ überschattet (2006-12-09).pretty.json\n", + "35999 - HSV-Sieg durch „Stinkefinger-Affäre“ überschattet (2006-12-09)\n", + "Processing datasets/german-quotations/train/39835 - Petra Pau: Schäuble ist „eifrigster Kämpfer gegen die Verfassung“ (2007-04-03).pretty.json\n", + "39835 - Petra Pau: Schäuble ist „eifrigster Kämpfer gegen die Verfassung“ (2007-04-03)\n", + "Processing datasets/german-quotations/train/66171 - Brandenburg: Entführung einer Vierjährigen (2011-02-11).pretty.json\n", + "66171 - Brandenburg: Entführung einer Vierjährigen (2011-02-11)\n", + "Processing datasets/german-quotations/train/48520 - Neues Grundrecht in Deutschland: Online-Durchsuchungen nur begrenzt möglich (2008-02-29).pretty.json\n", + "48520 - Neues Grundrecht in Deutschland: Online-Durchsuchungen nur begrenzt möglich (2008-02-29)\n", + "Processing datasets/german-quotations/train/72735 - Kriegsverbrecher Ratko Mladic steht vor dem UN-Tribunal in Den Haag (2012-05-17).pretty.json\n", + "72735 - Kriegsverbrecher Ratko Mladic steht vor dem UN-Tribunal in Den Haag (2012-05-17)\n", + "Processing datasets/german-quotations/train/73055 - Kenia: USA warnen von drohendem Anschlag (2012-06-24).pretty.json\n", + "73055 - Kenia: USA warnen von drohendem Anschlag (2012-06-24)\n", + "Processing datasets/german-quotations/train/52987 - Pannen beim CERN: LHC nach Stromausfall abgeschaltet (2008-09-19).pretty.json\n", + "52987 - Pannen beim CERN: LHC nach Stromausfall abgeschaltet (2008-09-19)\n", + "Processing datasets/german-quotations/train/33489 - Keine Verlängerung des Bundeswehreinsatzes im Kongo (2006-10-03).pretty.json\n", + "33489 - Keine Verlängerung des Bundeswehreinsatzes im Kongo (2006-10-03)\n", + "Processing datasets/german-quotations/train/3738 - Sgrena bezweifelt offizielle US-Angaben zum Beschuss des Fahrzeugs (2005-03-06).pretty.json\n", + "3738 - Sgrena bezweifelt offizielle US-Angaben zum Beschuss des Fahrzeugs (2005-03-06)\n", + "Processing datasets/german-quotations/train/18624 - Chaos Computer Club-Kongress in Berlin (2005-12-29).pretty.json\n", + "18624 - Chaos Computer Club-Kongress in Berlin (2005-12-29)\n", + "Processing datasets/german-quotations/train/18592 - Beziehung zwischen Indonesien und Timor-Leste stabilisieren sich (2005-12-29).pretty.json\n", + "18592 - Beziehung zwischen Indonesien und Timor-Leste stabilisieren sich (2005-12-29)\n", + "Processing datasets/german-quotations/train/55066 - Neues chinesisches Hybridauto auf dem Markt (2009-01-05).pretty.json\n", + "55066 - Neues chinesisches Hybridauto auf dem Markt (2009-01-05)\n", + "Processing datasets/german-quotations/train/67374 - Fukushima: 11.500 Tonnen radioaktiv verseuchtes Wasser sollen in den Pazifik abgelassen werden (2011-04-04).pretty.json\n", + "67374 - Fukushima: 11.500 Tonnen radioaktiv verseuchtes Wasser sollen in den Pazifik abgelassen werden (2011-04-04)\n", + "Processing datasets/german-quotations/train/31955 - UN-Generalsekretär kritisiert die Rekrutierung von Kindersoldaten im Sudan (2006-08-23).pretty.json\n", + "31955 - UN-Generalsekretär kritisiert die Rekrutierung von Kindersoldaten im Sudan (2006-08-23)\n", + "Processing datasets/german-quotations/train/47623 - Witten soll mit der „StadtGalerie“ ein neues Einkaufszentrum bekommen (2008-01-31).pretty.json\n", + "47623 - Witten soll mit der „StadtGalerie“ ein neues Einkaufszentrum bekommen (2008-01-31)\n", + "Processing datasets/german-quotations/train/49104 - Volker Beck stellt Strafanzeige gegen den Sänger „Bounty Killer“ (2008-03-20).pretty.json\n", + "49104 - Volker Beck stellt Strafanzeige gegen den Sänger „Bounty Killer“ (2008-03-20)\n", + "Processing datasets/german-quotations/train/71489 - Papua-Neuguinea: Untergang einer Fähre (2012-02-03).pretty.json\n", + "71489 - Papua-Neuguinea: Untergang einer Fähre (2012-02-03)\n", + "Processing datasets/german-quotations/train/54568 - Wiener Linien: zwischen eingeklemmt und abgestritten (2008-12-09).pretty.json\n", + "54568 - Wiener Linien: zwischen eingeklemmt und abgestritten (2008-12-09)\n", + "Processing datasets/german-quotations/train/60157 - Autobombe in Beirut zielt auf Hamas-Vertreter (2009-12-27).pretty.json\n", + "60157 - Autobombe in Beirut zielt auf Hamas-Vertreter (2009-12-27)\n", + "Processing datasets/german-quotations/train/66568 - UN-Sicherheitsrat beschließt Sanktionen gegen Libyen (2011-02-27).pretty.json\n", + "66568 - UN-Sicherheitsrat beschließt Sanktionen gegen Libyen (2011-02-27)\n", + "Processing datasets/german-quotations/train/15242 - Edmund Stoiber wird kein Regierungsmitglied (2005-11-01).pretty.json\n", + "15242 - Edmund Stoiber wird kein Regierungsmitglied (2005-11-01)\n", + "Processing datasets/german-quotations/train/42171 - Section-Control rechtswidrig (2007-06-15).pretty.json\n", + "42171 - Section-Control rechtswidrig (2007-06-15)\n", + "Processing datasets/german-quotations/train/37511 - Republik Kongo: 200 Gefangene aus einem Gefängnis ausgebrochen (2007-01-24).pretty.json\n", + "37511 - Republik Kongo: 200 Gefangene aus einem Gefängnis ausgebrochen (2007-01-24)\n", + "Processing datasets/german-quotations/train/74419 - Niederlande: 41-jähriger Schiedsrichter von Jugendlichen totgeprügelt (2012-12-05).pretty.json\n", + "74419 - Niederlande: 41-jähriger Schiedsrichter von Jugendlichen totgeprügelt (2012-12-05)\n", + "Processing datasets/german-quotations/train/5886 - Gedenkveranstaltung anlässlich des 16. Jahrestages des Tiananmen-Massakers (2005-06-05).pretty.json\n", + "5886 - Gedenkveranstaltung anlässlich des 16. Jahrestages des Tiananmen-Massakers (2005-06-05)\n", + "Processing datasets/german-quotations/train/3148 - Resistentes Aids-Virus entdeckt (2005-02-13).pretty.json\n", + "3148 - Resistentes Aids-Virus entdeckt (2005-02-13)\n", + "Processing datasets/german-quotations/train/24279 - Anklage gegen Hubschrauberpiloten von Sölden erhoben (2006-03-27).pretty.json\n", + "24279 - Anklage gegen Hubschrauberpiloten von Sölden erhoben (2006-03-27)\n", + "Processing datasets/german-quotations/train/44966 - Ölpreis knackt Marke von 90 US-Dollar (2007-10-19).pretty.json\n", + "44966 - Ölpreis knackt Marke von 90 US-Dollar (2007-10-19)\n", + "Processing datasets/german-quotations/train/23071 - Mannheim die erste Großstadt Deutschlands mit Vogelgrippe (2006-03-04).pretty.json\n", + "23071 - Mannheim die erste Großstadt Deutschlands mit Vogelgrippe (2006-03-04)\n", + "Processing datasets/german-quotations/train/35051 - Luxemburger Flaggenstreit erhitzt Gemüter (2006-11-16).pretty.json\n", + "35051 - Luxemburger Flaggenstreit erhitzt Gemüter (2006-11-16)\n", + "Processing datasets/german-quotations/train/17885 - Rücktritt des italienischen Notenbankchefs Fazio (2005-12-25).pretty.json\n", + "17885 - Rücktritt des italienischen Notenbankchefs Fazio (2005-12-25)\n", + "Processing datasets/german-quotations/train/48553 - Fast 300.000 Anträge für „.asia“-Domain (2008-03-02).pretty.json\n", + "48553 - Fast 300.000 Anträge für „.asia“-Domain (2008-03-02)\n", + "Processing datasets/german-quotations/train/75024 - Journalistin Tissy Bruns gestorben (2013-02-21).pretty.json\n", + "75024 - Journalistin Tissy Bruns gestorben (2013-02-21)\n", + "Processing datasets/german-quotations/train/33857 - Ban Ki-moon wird Nachfolger von UN-Generalsekretär Kofi Annan (2006-10-14).pretty.json\n", + "33857 - Ban Ki-moon wird Nachfolger von UN-Generalsekretär Kofi Annan (2006-10-14)\n", + "Processing datasets/german-quotations/train/46588 - Früherer Amtrak-Präsident George D. Warrington verstorben (2007-12-27).pretty.json\n", + "46588 - Früherer Amtrak-Präsident George D. Warrington verstorben (2007-12-27)\n", + "Processing datasets/german-quotations/train/74190 - Fritz Kuhn (GRÜNE) wird neuer Oberbürgermeister von Stuttgart (2012-10-22).pretty.json\n", + "74190 - Fritz Kuhn (GRÜNE) wird neuer Oberbürgermeister von Stuttgart (2012-10-22)\n", + "Processing datasets/german-quotations/train/49024 - Gewaltsame Niederschlagung der Proteste in Tibet befürchtet (2008-03-17).pretty.json\n", + "49024 - Gewaltsame Niederschlagung der Proteste in Tibet befürchtet (2008-03-17)\n", + "Processing datasets/german-quotations/train/18512 - Tupolew 214: Notlandung in Chabarowsk (2005-12-27).pretty.json\n", + "18512 - Tupolew 214: Notlandung in Chabarowsk (2005-12-27)\n", + "Processing datasets/german-quotations/train/54188 - Schäuble will Änderung der Abstimmungsregeln im Bundesrat (2008-11-23).pretty.json\n", + "54188 - Schäuble will Änderung der Abstimmungsregeln im Bundesrat (2008-11-23)\n", + "Processing datasets/german-quotations/train/4799 - Dschihad-Angehörige aus Untersuchungshaft geflohen (2005-04-22).pretty.json\n", + "4799 - Dschihad-Angehörige aus Untersuchungshaft geflohen (2005-04-22)\n", + "Processing datasets/german-quotations/train/13091 - Ehemann von Broadway-Star Bernadette Peters unter den Opfern beim Helikopterabsturz in Montenegro (2005-08-29).pretty.json\n", + "13091 - Ehemann von Broadway-Star Bernadette Peters unter den Opfern beim Helikopterabsturz in Montenegro (2005-08-29)\n", + "Processing datasets/german-quotations/train/84583 - Weiterhin extrem hohe Inflation in Argentinien beunruhigt die Regierung (2021-12-05).pretty.json\n", + "84583 - Weiterhin extrem hohe Inflation in Argentinien beunruhigt die Regierung (2021-12-05)\n", + "Processing datasets/german-quotations/train/15430 - Forschungsflugzeug abgestürzt - Polarforscher unverletzt (2005-11-03).pretty.json\n", + "15430 - Forschungsflugzeug abgestürzt - Polarforscher unverletzt (2005-11-03)\n", + "Processing datasets/german-quotations/train/48618 - Prinz Harry zurück auf britischem Boden (2008-03-02).pretty.json\n", + "48618 - Prinz Harry zurück auf britischem Boden (2008-03-02)\n", + "Processing datasets/german-quotations/train/48839 - Die beiden Nachbarplaneten der Erde sind sich ähnlicher als bisher angenommen (2008-03-09).pretty.json\n", + "48839 - Die beiden Nachbarplaneten der Erde sind sich ähnlicher als bisher angenommen (2008-03-09)\n", + "Processing datasets/german-quotations/train/63838 - Sonntagsfrage: linkes Lager klar vorne (2010-09-29).pretty.json\n", + "63838 - Sonntagsfrage: linkes Lager klar vorne (2010-09-29)\n", + "Processing datasets/german-quotations/train/21450 - IAEA einigt sich in der Iran-Frage auf Anrufung des Sicherheitsrates (2006-02-04).pretty.json\n", + "21450 - IAEA einigt sich in der Iran-Frage auf Anrufung des Sicherheitsrates (2006-02-04)\n", + "Processing datasets/german-quotations/train/36593 - Exoplaneten-Sucher „COROT“ ins All gestartet (2006-12-28).pretty.json\n", + "36593 - Exoplaneten-Sucher „COROT“ ins All gestartet (2006-12-28)\n", + "Processing datasets/german-quotations/train/61320 - Niger: Tausende demonstrieren für Machtübernahme durch das Militär (2010-02-20).pretty.json\n", + "61320 - Niger: Tausende demonstrieren für Machtübernahme durch das Militär (2010-02-20)\n", + "Processing datasets/german-quotations/train/48687 - Berliner Verkehrsbetriebe streiken ab 5. März 2008 (2008-03-05).pretty.json\n", + "48687 - Berliner Verkehrsbetriebe streiken ab 5. März 2008 (2008-03-05)\n", + "Processing datasets/german-quotations/train/33855 - Sechs Jahre OpenOffice.org (2006-10-14).pretty.json\n", + "33855 - Sechs Jahre OpenOffice.org (2006-10-14)\n", + "Processing datasets/german-quotations/train/56019 - Maria Riesch sorgt für den zweiten deutschen Weltmeistertitel im Slalom (2009-02-16).pretty.json\n", + "56019 - Maria Riesch sorgt für den zweiten deutschen Weltmeistertitel im Slalom (2009-02-16)\n", + "Processing datasets/german-quotations/train/14059 - Skytrain muss komplett saniert werden (2005-10-17).pretty.json\n", + "14059 - Skytrain muss komplett saniert werden (2005-10-17)\n", + "Processing datasets/german-quotations/train/47745 - Vereinigte Staaten: Präsident Bush plant massive Neuverschuldung 2009 (2008-02-04).pretty.json\n", + "47745 - Vereinigte Staaten: Präsident Bush plant massive Neuverschuldung 2009 (2008-02-04)\n", + "Processing datasets/german-quotations/train/44364 - Auschwitz-Bilderalbum von Paul Höcker veröffentlicht (2007-09-22).pretty.json\n", + "44364 - Auschwitz-Bilderalbum von Paul Höcker veröffentlicht (2007-09-22)\n", + "Processing datasets/german-quotations/train/69106 - Koch-Mehrin legt Widerspruch gegen Aberkennung des Doktortitels ein (2011-07-18).pretty.json\n", + "69106 - Koch-Mehrin legt Widerspruch gegen Aberkennung des Doktortitels ein (2011-07-18)\n", + "Processing datasets/german-quotations/train/44642 - The Rolling Stones – erfolgreichste Live-Band aller Zeiten (2007-10-05).pretty.json\n", + "44642 - The Rolling Stones – erfolgreichste Live-Band aller Zeiten (2007-10-05)\n", + "Processing datasets/german-quotations/train/37764 - In Europa gingen für fünf Minuten die Lichter aus (2007-02-03).pretty.json\n", + "37764 - In Europa gingen für fünf Minuten die Lichter aus (2007-02-03)\n", + "Processing datasets/german-quotations/train/63691 - CDU Baden-Württemberg boykottiert Abgeordnetenwatch (2010-09-06).pretty.json\n", + "63691 - CDU Baden-Württemberg boykottiert Abgeordnetenwatch (2010-09-06)\n", + "Processing datasets/german-quotations/train/40716 - Verbot „homosexueller Propaganda“ in Polen: EU-Kommissar Ján Figeľ im Interview mit Wikinews (2007-05-01).pretty.json\n", + "40716 - Verbot „homosexueller Propaganda“ in Polen: EU-Kommissar Ján Figeľ im Interview mit Wikinews (2007-05-01)\n", + "Processing datasets/german-quotations/train/67577 - Landesuntersuchungsamt warnt vor dem Schlankmacher „Reduce Weight Fruta Planta“ (2011-04-17).pretty.json\n", + "67577 - Landesuntersuchungsamt warnt vor dem Schlankmacher „Reduce Weight Fruta Planta“ (2011-04-17)\n", + "Processing datasets/german-quotations/train/59270 - Friedensnobelpreis geht an Obama (2009-10-09).pretty.json\n", + "59270 - Friedensnobelpreis geht an Obama (2009-10-09)\n", + "Processing datasets/german-quotations/train/77969 - Moody's stuft Sloweniens Rating hoch (2015-01-29).pretty.json\n", + "77969 - Moody's stuft Sloweniens Rating hoch (2015-01-29)\n", + "Processing datasets/german-quotations/train/5820 - Südamerika: Parlamentswahlen in Suriname (2005-06-02).pretty.json\n", + "5820 - Südamerika: Parlamentswahlen in Suriname (2005-06-02)\n", + "Processing datasets/german-quotations/train/45839 - Erneute Niederlage für Rechtsextremist Jürgen Rieger (2007-11-26).pretty.json\n", + "45839 - Erneute Niederlage für Rechtsextremist Jürgen Rieger (2007-11-26)\n", + "Processing datasets/german-quotations/train/77887 - Absatzanstieg bei Opel (2015-01-10).pretty.json\n", + "77887 - Absatzanstieg bei Opel (2015-01-10)\n", + "Processing datasets/german-quotations/train/9996 - Vierter Außeneinsatz am Spaceshuttle nicht nötig - Discovery landet Montag (2005-08-05).pretty.json\n", + "9996 - Vierter Außeneinsatz am Spaceshuttle nicht nötig - Discovery landet Montag (2005-08-05)\n", + "Processing datasets/german-quotations/train/5381 - Offenbar hohe Wahlbeteiligung bei der Parlamentswahl in Äthiopien (2005-05-16).pretty.json\n", + "5381 - Offenbar hohe Wahlbeteiligung bei der Parlamentswahl in Äthiopien (2005-05-16)\n", + "Processing datasets/german-quotations/train/22510 - Formel-1-Pilot Christian Klien weist Vorwurf der Körperverletzung zurück (2006-02-23).pretty.json\n", + "22510 - Formel-1-Pilot Christian Klien weist Vorwurf der Körperverletzung zurück (2006-02-23)\n", + "Processing datasets/german-quotations/train/80534 - Fußball: Zoff zwischen Leipzig und Dortmund (2017-02-10).pretty.json\n", + "80534 - Fußball: Zoff zwischen Leipzig und Dortmund (2017-02-10)\n", + "Processing datasets/german-quotations/train/8067 - Australier McEwen gewinnt seine zweite Etappe (2005-07-08).pretty.json\n", + "8067 - Australier McEwen gewinnt seine zweite Etappe (2005-07-08)\n", + "Processing datasets/german-quotations/train/80211 - Londoner Gericht verurteilt Cox' Mörder (2016-11-23).pretty.json\n", + "80211 - Londoner Gericht verurteilt Cox' Mörder (2016-11-23)\n", + "Processing datasets/german-quotations/train/9098 - Verdächtiger Terrorist im Londoner Stadtteil Stockwell festgenommen (2005-07-22).pretty.json\n", + "9098 - Verdächtiger Terrorist im Londoner Stadtteil Stockwell festgenommen (2005-07-22)\n", + "Processing datasets/german-quotations/train/51339 - Bündnis der Firmen Total und Enertrag zur Herstellung von Wasserstoff (2008-06-27).pretty.json\n", + "51339 - Bündnis der Firmen Total und Enertrag zur Herstellung von Wasserstoff (2008-06-27)\n", + "Processing datasets/german-quotations/train/54390 - Airbuswerke in Varel, Nordenham und Augsburg werden von EADS-Tochter Premium Aerotec weitergeführt (2008-11-29).pretty.json\n", + "54390 - Airbuswerke in Varel, Nordenham und Augsburg werden von EADS-Tochter Premium Aerotec weitergeführt (2008-11-29)\n", + "Processing datasets/german-quotations/train/61374 - EU-Kommission empfiehlt Beitrittsverhandlungen mit Island (2010-02-25).pretty.json\n", + "61374 - EU-Kommission empfiehlt Beitrittsverhandlungen mit Island (2010-02-25)\n", + "Processing datasets/german-quotations/train/12706 - Tropensturm Rita hat sich zum Hurrikan gewandelt (2005-09-21).pretty.json\n", + "12706 - Tropensturm Rita hat sich zum Hurrikan gewandelt (2005-09-21)\n", + "Processing datasets/german-quotations/train/82374 - Vierter „Chemtrail“ seit Jahresbeginn über Kaiserslautern (2018-04-06).pretty.json\n", + "82374 - Vierter „Chemtrail“ seit Jahresbeginn über Kaiserslautern (2018-04-06)\n", + "Processing datasets/german-quotations/train/56546 - Genfer Salon: Noch sind Elektroautos rar (2009-03-09).pretty.json\n", + "56546 - Genfer Salon: Noch sind Elektroautos rar (2009-03-09)\n", + "Processing datasets/german-quotations/train/60843 - US-Regierung will Atomkraft ausbauen (2010-02-01).pretty.json\n", + "60843 - US-Regierung will Atomkraft ausbauen (2010-02-01)\n", + "Processing datasets/german-quotations/train/4561 - Heftiger Streit um Religion als Pflichtfach (2005-04-10).pretty.json\n", + "4561 - Heftiger Streit um Religion als Pflichtfach (2005-04-10)\n", + "Processing datasets/german-quotations/train/44978 - Mutmaßlicher Kinderschänder festgenommen (2007-10-19).pretty.json\n", + "44978 - Mutmaßlicher Kinderschänder festgenommen (2007-10-19)\n", + "Processing datasets/german-quotations/train/69492 - Tegelbergbahn fährt nach Rettungsaktion wieder (2011-08-16).pretty.json\n", + "69492 - Tegelbergbahn fährt nach Rettungsaktion wieder (2011-08-16)\n", + "Processing datasets/german-quotations/train/51046 - Energieunternehmen „con energy“ investiert in Elektroauto „mindset“ (2008-06-19).pretty.json\n", + "51046 - Energieunternehmen „con energy“ investiert in Elektroauto „mindset“ (2008-06-19)\n", + "Processing datasets/german-quotations/train/10057 - Ermittler erheben Vorwürfe gegen Air-France-Piloten (2005-08-06).pretty.json\n", + "10057 - Ermittler erheben Vorwürfe gegen Air-France-Piloten (2005-08-06)\n", + "Processing datasets/german-quotations/train/25347 - Die Dominikanische Republik bekommt einen neuen Flughafen (2006-04-13).pretty.json\n", + "25347 - Die Dominikanische Republik bekommt einen neuen Flughafen (2006-04-13)\n", + "Processing datasets/german-quotations/train/4495 - Fürst Rainier von Monaco ist tot (2005-04-06).pretty.json\n", + "4495 - Fürst Rainier von Monaco ist tot (2005-04-06)\n", + "Processing datasets/german-quotations/train/28063 - Der Wechsel von Lukas Podolski zu Bayern München ist perfekt (2006-06-05).pretty.json\n", + "28063 - Der Wechsel von Lukas Podolski zu Bayern München ist perfekt (2006-06-05)\n", + "Processing datasets/german-quotations/train/925 - Neues Layout-Programm für Linux, Windows und OS X (2004-12-04).pretty.json\n", + "925 - Neues Layout-Programm für Linux, Windows und OS X (2004-12-04)\n", + "Processing datasets/german-quotations/train/30019 - Syd Barrett ist tot (2006-07-13).pretty.json\n", + "30019 - Syd Barrett ist tot (2006-07-13)\n", + "Processing datasets/german-quotations/train/73775 - Indien entwickelt neue Atomraketen (2012-09-02).pretty.json\n", + "73775 - Indien entwickelt neue Atomraketen (2012-09-02)\n", + "Processing datasets/german-quotations/train/34379 - Israelische Luftwaffe peilte deutsche Kriegsschiffe an (2006-10-29).pretty.json\n", + "34379 - Israelische Luftwaffe peilte deutsche Kriegsschiffe an (2006-10-29)\n", + "Processing datasets/german-quotations/train/60475 - Tausende Tote nach verheerendem Erdbeben in Haiti (2010-01-14).pretty.json\n", + "60475 - Tausende Tote nach verheerendem Erdbeben in Haiti (2010-01-14)\n", + "Processing datasets/german-quotations/train/82340 - Leipziger Buchmesse blickt nach Südosteuropa (2018-03-17).pretty.json\n", + "82340 - Leipziger Buchmesse blickt nach Südosteuropa (2018-03-17)\n", + "Processing datasets/german-quotations/train/20384 - Wikipedia.de außer Betrieb (2006-01-19).pretty.json\n", + "20384 - Wikipedia.de außer Betrieb (2006-01-19)\n", + "Processing datasets/german-quotations/train/5423 - Metager2 ist am Netz (2005-05-18).pretty.json\n", + "5423 - Metager2 ist am Netz (2005-05-18)\n", + "Processing datasets/german-quotations/dev/15274 - SPD-Krise: Auch Wieczorek-Zeul wirft das Handtuch (2005-11-01).pretty.json\n", + "15274 - SPD-Krise: Auch Wieczorek-Zeul wirft das Handtuch (2005-11-01)\n", + "Processing datasets/german-quotations/dev/33988 - In Bad Hersfeld erschallt wieder „Enner, zwoon, daäi – Bruder Lolls“ (2006-10-17).pretty.json\n", + "33988 - In Bad Hersfeld erschallt wieder „Enner, zwoon, daäi – Bruder Lolls“ (2006-10-17)\n", + "Processing datasets/german-quotations/dev/80023 - Hämische Kommentare im Netz über AirPods (2016-09-13).pretty.json\n", + "80023 - Hämische Kommentare im Netz über AirPods (2016-09-13)\n", + "Processing datasets/german-quotations/dev/76356 - Bremen hat die höchste Leistungsdichte bei Windkraft (2013-11-03).pretty.json\n", + "76356 - Bremen hat die höchste Leistungsdichte bei Windkraft (2013-11-03)\n", + "Processing datasets/german-quotations/dev/73035 - US-amerikanischer Deserteur gibt sich nach 28 Jahren in Schweden zu erkennen (2012-06-23).pretty.json\n", + "73035 - US-amerikanischer Deserteur gibt sich nach 28 Jahren in Schweden zu erkennen (2012-06-23)\n", + "Processing datasets/german-quotations/dev/25006 - Ur-Fisch in Kanada entdeckt (2006-04-07).pretty.json\n", + "25006 - Ur-Fisch in Kanada entdeckt (2006-04-07)\n", + "Processing datasets/german-quotations/dev/22959 - Ein Drittel der Arbeitsplätze bei Coca-Cola in Gefahr (2006-03-02).pretty.json\n", + "22959 - Ein Drittel der Arbeitsplätze bei Coca-Cola in Gefahr (2006-03-02)\n", + "Processing datasets/german-quotations/dev/6506 - Progress Raumfrachter dockt an Internationale Raumstation ISS an (2005-06-19).pretty.json\n", + "6506 - Progress Raumfrachter dockt an Internationale Raumstation ISS an (2005-06-19)\n", + "Processing datasets/german-quotations/dev/23741 - Dritthöchster Wasserfall der Welt entdeckt (2006-03-16).pretty.json\n", + "23741 - Dritthöchster Wasserfall der Welt entdeckt (2006-03-16)\n", + "Processing datasets/german-quotations/dev/64683 - Kritik an Nominierung von Schäuble-Tochter zur SWR-Fernsehfilmchefin (2010-11-19).pretty.json\n", + "64683 - Kritik an Nominierung von Schäuble-Tochter zur SWR-Fernsehfilmchefin (2010-11-19)\n", + "Processing datasets/german-quotations/dev/49037 - Finanzkrise – weltweit schwarzer Tag für die Börsen (2008-03-17).pretty.json\n", + "49037 - Finanzkrise – weltweit schwarzer Tag für die Börsen (2008-03-17)\n", + "Processing datasets/german-quotations/dev/57661 - USA bereiten sich auf „Cyberwar“ vor (2009-06-01).pretty.json\n", + "57661 - USA bereiten sich auf „Cyberwar“ vor (2009-06-01)\n", + "Processing datasets/german-quotations/dev/16470 - Japan: Daten über Erdbeben-Sicherheit gefälscht und nicht geprüft (2005-11-29).pretty.json\n", + "16470 - Japan: Daten über Erdbeben-Sicherheit gefälscht und nicht geprüft (2005-11-29)\n", + "Processing datasets/german-quotations/dev/67339 - Überschwemmungen in Thailand: Situation weiterhin kritisch (2011-04-02).pretty.json\n", + "67339 - Überschwemmungen in Thailand: Situation weiterhin kritisch (2011-04-02)\n", + "Processing datasets/german-quotations/dev/75792 - Unruhen im Westen Chinas: Dutzende Tote (2013-06-30).pretty.json\n", + "75792 - Unruhen im Westen Chinas: Dutzende Tote (2013-06-30)\n", + "Processing datasets/german-quotations/dev/53181 - Damenskispringen: Deutscher Doppelsieg in Oberstdorf (2008-09-26).pretty.json\n", + "53181 - Damenskispringen: Deutscher Doppelsieg in Oberstdorf (2008-09-26)\n", + "Processing datasets/german-quotations/dev/15510 - Bilanz der achten Nacht der Pariser Ausschreitungen (2005-11-04).pretty.json\n", + "15510 - Bilanz der achten Nacht der Pariser Ausschreitungen (2005-11-04)\n", + "Processing datasets/german-quotations/dev/53153 - Datenauslieferung an die Vereinigten Staaten: Der Arbeitskreis Vorratsdatenspeicherung veröffentlicht Geheimdokument (2008-09-25).pretty.json\n", + "53153 - Datenauslieferung an die Vereinigten Staaten: Der Arbeitskreis Vorratsdatenspeicherung veröffentlicht Geheimdokument (2008-09-25)\n", + "Processing datasets/german-quotations/dev/60632 - Google machte 2009 einen Gewinn von 4,6 Milliarden Euro (2010-01-23).pretty.json\n", + "60632 - Google machte 2009 einen Gewinn von 4,6 Milliarden Euro (2010-01-23)\n", + "Processing datasets/german-quotations/dev/39049 - Mönchengladbach: Nicht vollstreckter Haftbefehl ermöglichte Doppelmord (2007-03-16).pretty.json\n", + "39049 - Mönchengladbach: Nicht vollstreckter Haftbefehl ermöglichte Doppelmord (2007-03-16)\n", + "Processing datasets/german-quotations/dev/17060 - Rice-Erklärung zur CIA-Affäre: Fragen bleiben (2005-12-09).pretty.json\n", + "17060 - Rice-Erklärung zur CIA-Affäre: Fragen bleiben (2005-12-09)\n", + "Processing datasets/german-quotations/dev/67140 - Pipe Dreams - Eine Chronik des Lebens entlang der Pipeline (2011-03-23).pretty.json\n", + "67140 - Pipe Dreams - Eine Chronik des Lebens entlang der Pipeline (2011-03-23)\n", + "Processing datasets/german-quotations/dev/64195 - Frankreich: Borloo dementiert Gerüchte über Fillon-Nachfolge (2010-10-21).pretty.json\n", + "64195 - Frankreich: Borloo dementiert Gerüchte über Fillon-Nachfolge (2010-10-21)\n", + "Processing datasets/german-quotations/dev/58207 - G8-Gipfel: Nein zu Atomwaffen, Kritik an Iran (2009-07-10).pretty.json\n", + "58207 - G8-Gipfel: Nein zu Atomwaffen, Kritik an Iran (2009-07-10)\n", + "Processing datasets/german-quotations/dev/55038 - Israel startet Bodenoffensive in den Gazastreifen (2009-01-03).pretty.json\n", + "55038 - Israel startet Bodenoffensive in den Gazastreifen (2009-01-03)\n", + "Processing datasets/german-quotations/dev/2981 - US-Schauspieler Ossie Davis tot (2005-02-05).pretty.json\n", + "2981 - US-Schauspieler Ossie Davis tot (2005-02-05)\n", + "Processing datasets/german-quotations/dev/28114 - Bär wieder in Bayern (2006-06-06).pretty.json\n", + "28114 - Bär wieder in Bayern (2006-06-06)\n", + "Processing datasets/german-quotations/dev/48927 - Saturnsonde „Cassini“ passiert heute den Saturnmond Enceladus (2008-03-12).pretty.json\n", + "48927 - Saturnsonde „Cassini“ passiert heute den Saturnmond Enceladus (2008-03-12)\n", + "Processing datasets/german-quotations/dev/72093 - Starkes Erdbeben erschüttert Mexikos Süden (2012-03-21).pretty.json\n", + "72093 - Starkes Erdbeben erschüttert Mexikos Süden (2012-03-21)\n", + "Processing datasets/german-quotations/dev/58288 - Menden (Sauerland): Auto raste in Schützenumzug (2009-07-20).pretty.json\n", + "58288 - Menden (Sauerland): Auto raste in Schützenumzug (2009-07-20)\n", + "Processing datasets/german-quotations/dev/50288 - Taifun Halong tobte über die Philippinen (2008-05-19).pretty.json\n", + "50288 - Taifun Halong tobte über die Philippinen (2008-05-19)\n", + "Processing datasets/german-quotations/dev/82467 - Japan droht erneute Rezession (2018-06-11).pretty.json\n", + "82467 - Japan droht erneute Rezession (2018-06-11)\n", + "Processing datasets/german-quotations/dev/17905 - Gewerkschaft der Polizei kritisiert Innenminister Schäuble (2005-12-19).pretty.json\n", + "17905 - Gewerkschaft der Polizei kritisiert Innenminister Schäuble (2005-12-19)\n", + "Processing datasets/german-quotations/dev/16930 - Überraschung bei der ESA: Keine Beteiligung an russischer Raumfähre „Kliper“ beschlossen (2005-12-07).pretty.json\n", + "16930 - Überraschung bei der ESA: Keine Beteiligung an russischer Raumfähre „Kliper“ beschlossen (2005-12-07)\n", + "Processing datasets/german-quotations/dev/53717 - EU-Menschenrechtspreis an chinesischen Bürgerrechtler verliehen (2008-10-25).pretty.json\n", + "53717 - EU-Menschenrechtspreis an chinesischen Bürgerrechtler verliehen (2008-10-25)\n", + "Processing datasets/german-quotations/dev/33687 - Neuer Airbus-Chef Gallois will Arbeitsplätze abbauen (2006-10-10).pretty.json\n", + "33687 - Neuer Airbus-Chef Gallois will Arbeitsplätze abbauen (2006-10-10)\n", + "Processing datasets/german-quotations/dev/45056 - Friedensnobelpreisträger Al Gore hielt in Berlin eine Rede zum Klimaschutz (2007-10-24).pretty.json\n", + "45056 - Friedensnobelpreisträger Al Gore hielt in Berlin eine Rede zum Klimaschutz (2007-10-24)\n", + "Processing datasets/german-quotations/dev/64973 - Fluglotsenstreik in Spanien: Regierung ruft Alarmzustand aus (2010-12-04).pretty.json\n", + "64973 - Fluglotsenstreik in Spanien: Regierung ruft Alarmzustand aus (2010-12-04)\n", + "Processing datasets/german-quotations/dev/3216 - Deutsche Bank will auf betriebsbedingte Kündigungen verzichten (2005-02-15).pretty.json\n", + "3216 - Deutsche Bank will auf betriebsbedingte Kündigungen verzichten (2005-02-15)\n", + "Processing datasets/german-quotations/dev/33173 - Frankreich und Tunesien Weltmeister im Pétanque (2006-09-25).pretty.json\n", + "33173 - Frankreich und Tunesien Weltmeister im Pétanque (2006-09-25)\n", + "Processing datasets/german-quotations/dev/40463 - Erster südamerikanischer Energiegipfel tagt auf der Isla Margarita (2007-04-17).pretty.json\n", + "40463 - Erster südamerikanischer Energiegipfel tagt auf der Isla Margarita (2007-04-17)\n", + "Processing datasets/german-quotations/dev/7771 - SPD beschließt „Manifest“ zur Bundestagswahl (2005-07-04).pretty.json\n", + "7771 - SPD beschließt „Manifest“ zur Bundestagswahl (2005-07-04)\n", + "Processing datasets/german-quotations/dev/84711 - Australian Open: Nadal gewinnt das 21. Tennis-Grand-Slam-Turnier (2022-02-03).pretty.json\n", + "84711 - Australian Open: Nadal gewinnt das 21. Tennis-Grand-Slam-Turnier (2022-02-03)\n", + "Processing datasets/german-quotations/dev/8014 - Bundesrat beratschlagt über Korrekturen bei Hartz IV (2005-07-09).pretty.json\n", + "8014 - Bundesrat beratschlagt über Korrekturen bei Hartz IV (2005-07-09)\n", + "Processing datasets/german-quotations/dev/80698 - Frankreich: Mehrere Verletzte bei Schießerei in Schule (2017-03-17).pretty.json\n", + "80698 - Frankreich: Mehrere Verletzte bei Schießerei in Schule (2017-03-17)\n", + "Processing datasets/german-quotations/dev/52546 - Thailand: Trotz Verhängung des Ausnahmezustandes dauern Massenproteste an (2008-09-03).pretty.json\n", + "52546 - Thailand: Trotz Verhängung des Ausnahmezustandes dauern Massenproteste an (2008-09-03)\n", + "Processing datasets/german-quotations/dev/28924 - Nur noch 1.000 Dauerkarten beim FC St. Pauli (2006-06-24).pretty.json\n", + "28924 - Nur noch 1.000 Dauerkarten beim FC St. Pauli (2006-06-24)\n", + "Processing datasets/german-quotations/dev/78978 - Ein unpolitischer Abend für Kleinparteien in Bremen (2015-04-26).pretty.json\n", + "78978 - Ein unpolitischer Abend für Kleinparteien in Bremen (2015-04-26)\n", + "Processing datasets/german-quotations/dev/61850 - Vier neue Professoren in Maria Gugging (Österreich) (2010-03-26).pretty.json\n", + "61850 - Vier neue Professoren in Maria Gugging (Österreich) (2010-03-26)\n", + "Processing datasets/german-quotations/dev/9686 - Kundgebung von Mubarak-Gegnern in Kairo gewaltsam beendet (2005-07-31).pretty.json\n", + "9686 - Kundgebung von Mubarak-Gegnern in Kairo gewaltsam beendet (2005-07-31)\n", + "Processing datasets/german-quotations/dev/68176 - New York City: Strauss-Kahn bleibt in Untersuchungshaft (2011-05-16).pretty.json\n", + "68176 - New York City: Strauss-Kahn bleibt in Untersuchungshaft (2011-05-16)\n", + "Processing datasets/german-quotations/dev/40211 - Im VW-Werk Emden sollen bald auch Audi-Modelle vom Band rollen (2007-04-09).pretty.json\n", + "40211 - Im VW-Werk Emden sollen bald auch Audi-Modelle vom Band rollen (2007-04-09)\n", + "Processing datasets/german-quotations/dev/69005 - Krankenkassen: Mitglieder der geschlosenen City-BKK müssen bis 14. Juli eine Folgeversicherung vorweisen (2011-07-11).pretty.json\n", + "69005 - Krankenkassen: Mitglieder der geschlosenen City-BKK müssen bis 14. Juli eine Folgeversicherung vorweisen (2011-07-11)\n", + "Processing datasets/german-quotations/dev/52963 - Eklat während Wahlsendung – ödp-Spitzenkandidat Suttner verließ Gesprächsrunde wegen NPD vorzeitig (2008-09-17).pretty.json\n", + "52963 - Eklat während Wahlsendung – ödp-Spitzenkandidat Suttner verließ Gesprächsrunde wegen NPD vorzeitig (2008-09-17)\n", + "Processing datasets/german-quotations/dev/35359 - Kleinflugzeug stürzte auf Autobahn A 52 (2006-11-25).pretty.json\n", + "35359 - Kleinflugzeug stürzte auf Autobahn A 52 (2006-11-25)\n", + "Processing datasets/german-quotations/dev/34005 - Island nimmt kommerziellen Walfang wieder auf (2006-10-17).pretty.json\n", + "34005 - Island nimmt kommerziellen Walfang wieder auf (2006-10-17)\n", + "Processing datasets/german-quotations/dev/81491 - Die Freie Demokratische Partei kämpft um Wählerstimmen (2017-08-29).pretty.json\n", + "81491 - Die Freie Demokratische Partei kämpft um Wählerstimmen (2017-08-29)\n", + "Processing datasets/german-quotations/dev/36276 - Kamel auf türkischem Flughafen geopfert (2006-12-16).pretty.json\n", + "36276 - Kamel auf türkischem Flughafen geopfert (2006-12-16)\n", + "Processing datasets/german-quotations/dev/84153 - Friedensnobelpreis 2020 geht an das Welternährungsprogramm der Vereinten Nationen (2020-10-10).pretty.json\n", + "84153 - Friedensnobelpreis 2020 geht an das Welternährungsprogramm der Vereinten Nationen (2020-10-10)\n", + "Processing datasets/german-quotations/dev/38981 - Madrid: Denkmal für die Terroropfer von 2004 eingeweiht (2007-03-12).pretty.json\n", + "38981 - Madrid: Denkmal für die Terroropfer von 2004 eingeweiht (2007-03-12)\n", + "Processing datasets/german-quotations/dev/71241 - Tschechischer Künstler will 60.000 Exemplare von Sarrazins Buch recyceln (2012-01-15).pretty.json\n", + "71241 - Tschechischer Künstler will 60.000 Exemplare von Sarrazins Buch recyceln (2012-01-15)\n", + "Processing datasets/german-quotations/dev/16696 - West-Papua: Häftling hisst „Morning Star“ (2005-12-03).pretty.json\n", + "16696 - West-Papua: Häftling hisst „Morning Star“ (2005-12-03)\n", + "Processing datasets/german-quotations/dev/43859 - Neubau des Jenoptik-Betriebskindergartens eröffnet (2007-08-28).pretty.json\n", + "43859 - Neubau des Jenoptik-Betriebskindergartens eröffnet (2007-08-28)\n", + "Processing datasets/german-quotations/dev/27603 - Internationale Truppe soll Lage in Timor-Leste stabilisieren (2006-05-27).pretty.json\n", + "27603 - Internationale Truppe soll Lage in Timor-Leste stabilisieren (2006-05-27)\n", + "Processing datasets/german-quotations/dev/71370 - Mexikanischer Präsident verkündet Notfallplan wegen extremer Trockenheit (2012-01-24).pretty.json\n", + "71370 - Mexikanischer Präsident verkündet Notfallplan wegen extremer Trockenheit (2012-01-24)\n", + "Processing datasets/german-quotations/dev/38971 - Deutsche Verbraucher kaufen mehr, aber billigere Schuhe (2007-03-12).pretty.json\n", + "38971 - Deutsche Verbraucher kaufen mehr, aber billigere Schuhe (2007-03-12)\n", + "Processing datasets/german-quotations/dev/74519 - Volksrepublik China: Hochgeschwindigkeitsbahnstrecke Peking–Guangzhou eröffnet (2012-12-28).pretty.json\n", + "74519 - Volksrepublik China: Hochgeschwindigkeitsbahnstrecke Peking–Guangzhou eröffnet (2012-12-28)\n", + "Processing datasets/german-quotations/dev/36485 - Jimmy Wales sagt Google mit Suchmaschine den Kampf an (2006-12-24).pretty.json\n", + "36485 - Jimmy Wales sagt Google mit Suchmaschine den Kampf an (2006-12-24)\n", + "Processing datasets/german-quotations/dev/29368 - Ingeborg-Bachmann-Preis 2006 ging an Kathrin Passig (2006-07-01).pretty.json\n", + "29368 - Ingeborg-Bachmann-Preis 2006 ging an Kathrin Passig (2006-07-01)\n", + "Processing datasets/german-quotations/dev/45353 - Mexikanischer Bundesstaat Tabasco überflutet (2007-11-03).pretty.json\n", + "45353 - Mexikanischer Bundesstaat Tabasco überflutet (2007-11-03)\n", + "Processing datasets/german-quotations/dev/5352 - Familiendrama in Witten (2005-05-14).pretty.json\n", + "5352 - Familiendrama in Witten (2005-05-14)\n", + "Processing datasets/german-quotations/dev/31406 - Hoffnung auf Waffenstillstand in Nahost (2006-08-13).pretty.json\n", + "31406 - Hoffnung auf Waffenstillstand in Nahost (2006-08-13)\n", + "Processing datasets/german-quotations/dev/48004 - Unabhängigkeitserklärung des Kosovos am 17. Februar 2008 erwartet (2008-02-15).pretty.json\n", + "48004 - Unabhängigkeitserklärung des Kosovos am 17. Februar 2008 erwartet (2008-02-15)\n", + "Processing datasets/german-quotations/dev/52532 - ARGE versetzt Mutter zweier Kinder in Angst und Schrecken (2008-09-02).pretty.json\n", + "52532 - ARGE versetzt Mutter zweier Kinder in Angst und Schrecken (2008-09-02)\n", + "Processing datasets/german-quotations/dev/61440 - Orkantief Xynthia wütet über Europa (2010-03-01).pretty.json\n", + "61440 - Orkantief Xynthia wütet über Europa (2010-03-01)\n", + "Processing datasets/german-quotations/dev/65535 - Berlin: Parteitag der islamkritischen Partei „Die Freiheit“ konnte nicht stattfinden (2011-01-12).pretty.json\n", + "65535 - Berlin: Parteitag der islamkritischen Partei „Die Freiheit“ konnte nicht stattfinden (2011-01-12)\n", + "Processing datasets/german-quotations/dev/76866 - Massaker im Südsudan: Hunderte Zivilisten getötet (2014-04-22).pretty.json\n", + "76866 - Massaker im Südsudan: Hunderte Zivilisten getötet (2014-04-22)\n", + "Processing datasets/german-quotations/dev/61671 - EADS verliert Milliardenauftrag für US-Luftwaffe (2010-03-09).pretty.json\n", + "61671 - EADS verliert Milliardenauftrag für US-Luftwaffe (2010-03-09)\n", + "Processing datasets/german-quotations/dev/30088 - Luxemburg: Zahlreiche Verletzte bei Brand eines Zugwaggons (2006-07-15).pretty.json\n", + "30088 - Luxemburg: Zahlreiche Verletzte bei Brand eines Zugwaggons (2006-07-15)\n", + "Processing datasets/german-quotations/dev/79539 - Indoktrination in islamischen Kindergärten aufgedeckt (2016-02-29).pretty.json\n", + "79539 - Indoktrination in islamischen Kindergärten aufgedeckt (2016-02-29)\n", + "Processing datasets/german-quotations/dev/74355 - Tanklager in Bremen-Farge: Zukunft noch ungewiss (2012-11-20).pretty.json\n", + "74355 - Tanklager in Bremen-Farge: Zukunft noch ungewiss (2012-11-20)\n", + "Processing datasets/german-quotations/dev/49380 - Neapels Müll wird jetzt in Deutschland entsorgt (2008-04-01).pretty.json\n", + "49380 - Neapels Müll wird jetzt in Deutschland entsorgt (2008-04-01)\n", + "Processing datasets/german-quotations/dev/39256 - Vollsperrung der A 2 nach Gefahrgutunfall (2007-03-23).pretty.json\n", + "39256 - Vollsperrung der A 2 nach Gefahrgutunfall (2007-03-23)\n", + "Processing datasets/german-quotations/dev/73255 - Dortmund: Streit zwischen zwei Familien endet fast tödlich (2012-07-05).pretty.json\n", + "73255 - Dortmund: Streit zwischen zwei Familien endet fast tödlich (2012-07-05)\n", + "Processing datasets/german-quotations/dev/64906 - Feindliche Übernahme von Hochtief durch ACS steht offenbar unmittelbar bevor (2010-12-01).pretty.json\n", + "64906 - Feindliche Übernahme von Hochtief durch ACS steht offenbar unmittelbar bevor (2010-12-01)\n", + "Processing datasets/german-quotations/dev/17214 - Über 100 Tote bei Flugzeugabsturz in Nigeria (2005-12-10).pretty.json\n", + "17214 - Über 100 Tote bei Flugzeugabsturz in Nigeria (2005-12-10)\n", + "Processing datasets/german-quotations/dev/48343 - Erdbeben erschüttert England (2008-02-27).pretty.json\n", + "48343 - Erdbeben erschüttert England (2008-02-27)\n", + "Processing datasets/german-quotations/dev/49475 - Simbabwe: Gericht nimmt Klage auf Veröffentlichung der Wahlergebnisse zur Entscheidung an (2008-04-07).pretty.json\n", + "49475 - Simbabwe: Gericht nimmt Klage auf Veröffentlichung der Wahlergebnisse zur Entscheidung an (2008-04-07)\n", + "Processing datasets/german-quotations/dev/69844 - Oktoberfest: Betrüger verkaufen falsche Tischreservierungen (2011-09-26).pretty.json\n", + "69844 - Oktoberfest: Betrüger verkaufen falsche Tischreservierungen (2011-09-26)\n", + "Processing datasets/german-quotations/dev/37752 - Raubüberfall in Kaiserslautern: 20-Jähriger zusammengeschlagen und bestohlen (2007-02-02).pretty.json\n", + "37752 - Raubüberfall in Kaiserslautern: 20-Jähriger zusammengeschlagen und bestohlen (2007-02-02)\n", + "Processing datasets/german-quotations/dev/13389 - Sohn von Willy Bogner hat Selbstmord begangen (2005-10-04).pretty.json\n", + "13389 - Sohn von Willy Bogner hat Selbstmord begangen (2005-10-04)\n", + "Processing datasets/german-quotations/dev/48555 - Fußball: Marco van Basten ist neuer Trainer von Ajax Amsterdam (2008-02-29).pretty.json\n", + "48555 - Fußball: Marco van Basten ist neuer Trainer von Ajax Amsterdam (2008-02-29)\n", + "Processing datasets/german-quotations/dev/84747 - Mannheim: Ukraine-Friedensmarsch überquert Rhein und Neckar (2022-03-06).pretty.json\n", + "84747 - Mannheim: Ukraine-Friedensmarsch überquert Rhein und Neckar (2022-03-06)\n", + "Processing datasets/german-quotations/dev/76796 - G8 der Industrie schließen Russland aus und sorgen damit für Diskussionen auch beim Atomgipfel (2014-03-27).pretty.json\n", + "76796 - G8 der Industrie schließen Russland aus und sorgen damit für Diskussionen auch beim Atomgipfel (2014-03-27)\n", + "Processing datasets/german-quotations/dev/75087 - EU-Grenzwerte überschritten: Keine Fristverlängerung für deutsche Städte (2013-03-02).pretty.json\n", + "75087 - EU-Grenzwerte überschritten: Keine Fristverlängerung für deutsche Städte (2013-03-02)\n", + "Processing datasets/german-quotations/dev/38693 - Kanadisches Parlament lehnt Verlängerung der Anti-Terror-Gesetze ab (2007-03-01).pretty.json\n", + "38693 - Kanadisches Parlament lehnt Verlängerung der Anti-Terror-Gesetze ab (2007-03-01)\n", + "Processing datasets/german-quotations/dev/60617 - Kalifornien: Notstand wegen Stürmen und starker Regenfälle ausgerufen (2010-01-22).pretty.json\n", + "60617 - Kalifornien: Notstand wegen Stürmen und starker Regenfälle ausgerufen (2010-01-22)\n", + "Processing datasets/german-quotations/dev/30032 - Bürgerrechte werden weiter aufgeweicht – Regierung plant Ausweitung der Anti-Terror-Gesetze (2006-07-13).pretty.json\n", + "30032 - Bürgerrechte werden weiter aufgeweicht – Regierung plant Ausweitung der Anti-Terror-Gesetze (2006-07-13)\n", + "Processing datasets/german-quotations/dev/80634 - Guatemala: 22 Mädchen sterben nach Brandstiftung in einem Waisenhaus (2017-03-09).pretty.json\n", + "80634 - Guatemala: 22 Mädchen sterben nach Brandstiftung in einem Waisenhaus (2017-03-09)\n", + "Processing datasets/german-quotations/dev/82573 - Sexualstraftäter am Campus der Uni Frankfurt verurteilt (2018-08-20).pretty.json\n", + "82573 - Sexualstraftäter am Campus der Uni Frankfurt verurteilt (2018-08-20)\n", + "Processing datasets/german-quotations/dev/27575 - US-Soldaten begehen Kriegsverbrechen an irakischen Zivilisten (2006-05-28).pretty.json\n", + "27575 - US-Soldaten begehen Kriegsverbrechen an irakischen Zivilisten (2006-05-28)\n", + "Processing datasets/german-quotations/dev/65670 - München: Altes Bauernhaus gerät in Brand (2011-01-18).pretty.json\n", + "65670 - München: Altes Bauernhaus gerät in Brand (2011-01-18)\n", + "Processing datasets/german-quotations/dev/40205 - Thailand: Test des Tsunamiwarnsystems löste Panik aus (2007-04-09).pretty.json\n", + "40205 - Thailand: Test des Tsunamiwarnsystems löste Panik aus (2007-04-09)\n", + "Processing datasets/german-quotations/dev/22544 - Schweizer qualifizieren sich als Gruppenzweiter für die olympischen Viertelfinale (2006-02-22).pretty.json\n", + "22544 - Schweizer qualifizieren sich als Gruppenzweiter für die olympischen Viertelfinale (2006-02-22)\n", + "Processing datasets/german-quotations/dev/47942 - Huckabee gewinnt Republikaner-Vorwahl in Kansas (2008-02-11).pretty.json\n", + "47942 - Huckabee gewinnt Republikaner-Vorwahl in Kansas (2008-02-11)\n", + "Processing datasets/german-quotations/dev/42503 - Das neue Zebra-Pferdchen (2007-07-02).pretty.json\n", + "42503 - Das neue Zebra-Pferdchen (2007-07-02)\n", + "Processing datasets/german-quotations/dev/69650 - Pakistan: Mehr als 20 Tote bei Selbstmordattentat (2011-09-08).pretty.json\n", + "69650 - Pakistan: Mehr als 20 Tote bei Selbstmordattentat (2011-09-08)\n", + "Processing datasets/german-quotations/dev/29336 - Nach Einbürgerungsstreit um Ayaan Hirsi Ali: Balkenende zurückgetreten (2006-06-30).pretty.json\n", + "29336 - Nach Einbürgerungsstreit um Ayaan Hirsi Ali: Balkenende zurückgetreten (2006-06-30)\n", + "Processing datasets/german-quotations/dev/68734 - Deutschland: Regierung rüstet auf im Kampf gegen Cyberattacken aus dem Internet (2011-06-16).pretty.json\n", + "68734 - Deutschland: Regierung rüstet auf im Kampf gegen Cyberattacken aus dem Internet (2011-06-16)\n", + "Processing datasets/german-quotations/dev/13837 - Erster Missionstag von „Shenzhou 6“: Taikonaut stieg in Orbitalmodul um (2005-10-13).pretty.json\n", + "13837 - Erster Missionstag von „Shenzhou 6“: Taikonaut stieg in Orbitalmodul um (2005-10-13)\n", + "Processing datasets/german-quotations/dev/5906 - Zeitgenössische Kunstausstellung auf der Festung Ehrenbreitstein eröffnet (2005-06-05).pretty.json\n", + "5906 - Zeitgenössische Kunstausstellung auf der Festung Ehrenbreitstein eröffnet (2005-06-05)\n", + "Processing datasets/german-quotations/dev/61652 - Rechtspopulist Wilders zieht in den Stadtrat von Den Haag ein (2010-03-09).pretty.json\n", + "61652 - Rechtspopulist Wilders zieht in den Stadtrat von Den Haag ein (2010-03-09)\n", + "Processing datasets/german-quotations/dev/44604 - Speyer: Im Krankentransportwagen verstorben (2007-10-05).pretty.json\n", + "44604 - Speyer: Im Krankentransportwagen verstorben (2007-10-05)\n", + "Processing datasets/german-quotations/dev/56334 - Billigfluggesellschaft Ryanair erwägt Toilettenbenutzungsgebühr (2009-03-01).pretty.json\n", + "56334 - Billigfluggesellschaft Ryanair erwägt Toilettenbenutzungsgebühr (2009-03-01)\n", + "Processing datasets/german-quotations/dev/80019 - Verdächtiger Salafist in Niedersachsen verschwunden (2016-09-10).pretty.json\n", + "80019 - Verdächtiger Salafist in Niedersachsen verschwunden (2016-09-10)\n", + "Processing datasets/german-quotations/dev/72867 - Schlecker-Insolvenz: Verkäuferinnen zu Erzieherinnen und Altenpflegerinnen umschulen (2012-06-08).pretty.json\n", + "72867 - Schlecker-Insolvenz: Verkäuferinnen zu Erzieherinnen und Altenpflegerinnen umschulen (2012-06-08)\n", + "Processing datasets/german-quotations/dev/29847 - Verunglückter Marinetaucher war bekannter Schwimmer (2006-07-08).pretty.json\n", + "29847 - Verunglückter Marinetaucher war bekannter Schwimmer (2006-07-08)\n", + "Processing datasets/german-quotations/dev/65838 - Patent: Motorwagen Nr. 1 von Carl Benz (2011-01-29).pretty.json\n", + "65838 - Patent: Motorwagen Nr. 1 von Carl Benz (2011-01-29)\n", + "Processing datasets/german-quotations/dev/47821 - Für 200 Leute stand die Zeit still (2008-02-06).pretty.json\n", + "47821 - Für 200 Leute stand die Zeit still (2008-02-06)\n", + "Processing datasets/german-quotations/dev/11059 - Passagierflugzeug landete in Guam mit defektem Fahrwerk (2005-08-20).pretty.json\n", + "11059 - Passagierflugzeug landete in Guam mit defektem Fahrwerk (2005-08-20)\n", + "Processing datasets/german-quotations/dev/6028 - Äthiopien: Tote bei Protesten gegen die Regierung (2005-06-18).pretty.json\n", + "6028 - Äthiopien: Tote bei Protesten gegen die Regierung (2005-06-18)\n", + "Processing datasets/german-quotations/dev/76695 - Der Physiker Sebastian Pflugbeil berichtet aus Fukushima (2014-02-08).pretty.json\n", + "76695 - Der Physiker Sebastian Pflugbeil berichtet aus Fukushima (2014-02-08)\n", + "Processing datasets/german-quotations/dev/26056 - Tankstellenangestellte aus Greifswald wurde ermordet (2006-05-01).pretty.json\n", + "26056 - Tankstellenangestellte aus Greifswald wurde ermordet (2006-05-01)\n", + "Processing datasets/german-quotations/dev/2745 - Festnahme von zwei mutmaßlichen „Al-Qaida“-Mitgliedern in Mainz (2005-01-23).pretty.json\n", + "2745 - Festnahme von zwei mutmaßlichen „Al-Qaida“-Mitgliedern in Mainz (2005-01-23)\n", + "Processing datasets/german-quotations/dev/4895 - Jungfernflug des A380 soll am kommenden Mittwoch stattfinden (2005-04-25).pretty.json\n", + "4895 - Jungfernflug des A380 soll am kommenden Mittwoch stattfinden (2005-04-25)\n", + "Processing datasets/german-quotations/dev/34359 - Illegale Geschäfte mit Gewerbemüll auf einer Mülldeponie bei Schönberg? (2006-10-28).pretty.json\n", + "34359 - Illegale Geschäfte mit Gewerbemüll auf einer Mülldeponie bei Schönberg? (2006-10-28)\n", + "Processing datasets/german-quotations/dev/51806 - Serbien: Mutmaßlicher Kriegsverbrecher Radovan Karadžić gefasst (2008-07-22).pretty.json\n", + "51806 - Serbien: Mutmaßlicher Kriegsverbrecher Radovan Karadžić gefasst (2008-07-22)\n", + "Processing datasets/german-quotations/dev/78997 - Internationaler Gedenktag für Rashid Rehman (2015-05-08).pretty.json\n", + "78997 - Internationaler Gedenktag für Rashid Rehman (2015-05-08)\n", + "Processing datasets/german-quotations/dev/44357 - Myanmar: Mönche protestieren weiter (2007-09-20).pretty.json\n", + "44357 - Myanmar: Mönche protestieren weiter (2007-09-20)\n", + "Processing datasets/german-quotations/dev/6620 - Hansa Rostock verpflichtet Oumar Kondé (2005-06-21).pretty.json\n", + "6620 - Hansa Rostock verpflichtet Oumar Kondé (2005-06-21)\n", + "Processing datasets/german-quotations/dev/58969 - Bald neuer Jackson Song (2009-09-26).pretty.json\n", + "58969 - Bald neuer Jackson Song (2009-09-26)\n", + "Processing datasets/german-quotations/dev/8814 - George Hincapie gewinnt fünfzehnte Touretappe (2005-07-17).pretty.json\n", + "8814 - George Hincapie gewinnt fünfzehnte Touretappe (2005-07-17)\n", + "Processing datasets/german-quotations/dev/75720 - Tödliches Busfeuer von Selbstmörder verursacht (2013-06-09).pretty.json\n", + "75720 - Tödliches Busfeuer von Selbstmörder verursacht (2013-06-09)\n", + "Processing datasets/german-quotations/dev/18058 - Mannesmannprozess wird neu aufgerollt (2005-12-21).pretty.json\n", + "18058 - Mannesmannprozess wird neu aufgerollt (2005-12-21)\n", + "Processing datasets/german-quotations/dev/26888 - Porno-Domain: Sprecher von EU-Medienkommissarin Viviane Reding sprach von politischer Einflussnahme (2006-05-15).pretty.json\n", + "26888 - Porno-Domain: Sprecher von EU-Medienkommissarin Viviane Reding sprach von politischer Einflussnahme (2006-05-15)\n", + "Processing datasets/german-quotations/dev/58562 - Deutsche Pharmafirma soll mit Leichenteilen aus der Ukraine handeln (2009-08-24).pretty.json\n", + "58562 - Deutsche Pharmafirma soll mit Leichenteilen aus der Ukraine handeln (2009-08-24)\n", + "Processing datasets/german-quotations/dev/77143 - Putin wendet sich an China für Rüstungsgüter (2014-08-07).pretty.json\n", + "77143 - Putin wendet sich an China für Rüstungsgüter (2014-08-07)\n", + "Processing datasets/german-quotations/dev/75244 - Suhl in Thüringen: Geiselnahme im Gefängnis durch Polizei beendet (2013-03-30).pretty.json\n", + "75244 - Suhl in Thüringen: Geiselnahme im Gefängnis durch Polizei beendet (2013-03-30)\n", + "Processing datasets/german-quotations/dev/33097 - Flugunfall im Kreis Ostholstein (2006-09-24).pretty.json\n", + "33097 - Flugunfall im Kreis Ostholstein (2006-09-24)\n", + "Processing datasets/german-quotations/dev/32238 - Blitzeinschlag bei Flugschau in Hangelar (2006-08-28).pretty.json\n", + "32238 - Blitzeinschlag bei Flugschau in Hangelar (2006-08-28)\n", + "Processing datasets/german-quotations/dev/50777 - Tōkyō: Amokläufer tötet sieben Menschen (2008-06-08).pretty.json\n", + "50777 - Tōkyō: Amokläufer tötet sieben Menschen (2008-06-08)\n", + "Processing datasets/german-quotations/dev/3855 - Landser laut BGH eine kriminelle Vereinigung (2005-03-10).pretty.json\n", + "3855 - Landser laut BGH eine kriminelle Vereinigung (2005-03-10)\n", + "Processing datasets/german-quotations/dev/60162 - Deutschsprachige Wikipedia feiert 1.000.000. Artikel (2009-12-27).pretty.json\n", + "60162 - Deutschsprachige Wikipedia feiert 1.000.000. Artikel (2009-12-27)\n", + "Processing datasets/german-quotations/dev/12324 - Jenaer Feuerteufel gefasst (2005-09-10).pretty.json\n", + "12324 - Jenaer Feuerteufel gefasst (2005-09-10)\n", + "Processing datasets/german-quotations/dev/70626 - Größtes Flusskraftwerk in Norddeutschland eingeweiht (2011-11-30).pretty.json\n", + "70626 - Größtes Flusskraftwerk in Norddeutschland eingeweiht (2011-11-30)\n", + "Processing datasets/german-quotations/dev/58138 - Greenpeace-Aktivisten „schließen“ Atomkraftwerk Krümmel (2009-07-06).pretty.json\n", + "58138 - Greenpeace-Aktivisten „schließen“ Atomkraftwerk Krümmel (2009-07-06)\n", + "Processing datasets/german-quotations/dev/28234 - Irakische Regierung besetzte Innen- und Verteidigungsministerium (2006-06-08).pretty.json\n", + "28234 - Irakische Regierung besetzte Innen- und Verteidigungsministerium (2006-06-08)\n", + "Processing datasets/german-quotations/dev/52410 - Unruhen in Kaschmir halten an (2008-09-05).pretty.json\n", + "52410 - Unruhen in Kaschmir halten an (2008-09-05)\n", + "Processing datasets/german-quotations/dev/11969 - Viktoriasee in Uganda mit toxischen Stoffen belastet (2005-09-06).pretty.json\n", + "11969 - Viktoriasee in Uganda mit toxischen Stoffen belastet (2005-09-06)\n", + "Processing datasets/german-quotations/dev/77331 - Entscheidung zu neuem Wahlrecht verschärft Konflikt in Hongkong (2014-08-02).pretty.json\n", + "77331 - Entscheidung zu neuem Wahlrecht verschärft Konflikt in Hongkong (2014-08-02)\n" + ] + } + ], + "source": [ + "result = []\n", + "for dataset_path in paths:\n", + " for file in dataset_path.glob(\"*.json\"):\n", + " with open(file, \"r\") as f:\n", + " data = json.load(f)\n", + " \n", + "\n", + " print(f\"Processing {file}\")\n", + " print(data[\"documentName\"])\n", + "\n", + " # get the whole document as tokens\n", + " document_tokens = [token for sentence in data[\"sentences\"] for token in sentence[\"tokens\"]]\n", + " document_token_ids = [token_id for sentence in data[\"sentences\"] for token_id in sentence[\"tokenIds\"]]\n", + " document_token_id2token = {token_id: token for token_id, token in zip(document_token_ids, document_tokens)}\n", + " assert len(document_tokens) == len(document_token_ids), \"Lengths do not match\"\n", + "\n", + " # get the speaker tokens\n", + " speaker_token_ids = [token_id for annotation in data[\"annotations\"] for token_id in annotation[\"speaker\"][\"tokenIds\"] if annotation[\"type\"] == \"Direct\"]\n", + "\n", + " # get the quotation tokens\n", + " quote_token_ids = [token_id for annotation in data[\"annotations\"] for token_id in annotation[\"quote\"][\"tokenIds\"] if annotation[\"type\"] == \"Direct\"]\n", + " # quote_types = [annotation[\"type\"] for annotation in data[\"annotations\"] for token_id in annotation[\"quote\"][\"tokenIds\"]]\n", + "\n", + " # sanity checks\n", + " # I assume that speakers do not overlap\n", + " # assert len(speaker_token_ids) == len(set(speaker_token_ids)), \"Speakers overlap\"\n", + "\n", + " # I assume that quotes do not overlap\n", + " # assert len(quote_token_ids) == len(set(quote_token_ids)), \"Quotes overlap\"\n", + "\n", + " # I assume that speakers and quotes do not overlap\n", + " assert len(set(speaker_token_ids).intersection(set(quote_token_ids))) == 0, \"Speakers and quotes overlap\"\n", + "\n", + " # build dicts of annotated tokens and their class\n", + " # coarse-grained\n", + " annotated_tokens = {\n", + " token_id: \"speaker\" for token_id in speaker_token_ids\n", + " }\n", + " annotated_tokens.update({\n", + " token_id: \"quote\" for token_id in quote_token_ids\n", + " })\n", + "\n", + " # fine-grained \n", + " # annotated_tokens_fine = {\n", + " # token_id: \"speaker\" for token_id in speaker_token_ids\n", + " # }\n", + " # annotated_tokens_fine.update({\n", + " # token_id: f\"quote-{token_type.lower()}\" for token_id, token_type in zip(quote_token_ids, quote_types)\n", + " # })\n", + "\n", + " # create the sequence classification data\n", + " tags = []\n", + " # tags_fine = []\n", + " for token_id in document_token_ids:\n", + " if token_id in annotated_tokens:\n", + " tags.append(annotated_tokens[token_id])\n", + " # tags_fine.append(annotated_tokens_fine[token_id])\n", + " else:\n", + " tags.append(\"O\")\n", + " # tags_fine.append(\"O\")\n", + "\n", + " # build result object\n", + " result.append({\n", + " \"tokens\": document_tokens,\n", + " \"tags\": tags,\n", + " # \"tags_fine\": tags_fine\n", + " })\n", + "\n", + "# write the result\n", + "df = pd.DataFrame(result)\n", + "df.to_parquet(\"datasets/german-quotations/german_quotations_test.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data[\"documentName\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "document_tokens = [token for sentence in data[\"sentences\"] for token in sentence[\"tokens\"]]\n", + "document_token_ids = [token_id for sentence in data[\"sentences\"] for token_id in sentence[\"tokenIds\"]]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "assert len(document_tokens) == len(document_token_ids), \"Lengths do not match\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "document_token_id2token = {token_id: token for token_id, token in zip(document_token_ids, document_tokens)}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "speaker_token_ids = [token_id for annotation in data[\"annotations\"] for token_id in annotation[\"speaker\"][\"tokenIds\"]]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "speaker_token_ids" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "quote_token_ids = [token_id for annotation in data[\"annotations\"] for token_id in annotation[\"quote\"][\"tokenIds\"]]\n", + "quote_types = [annotation[\"type\"] for annotation in data[\"annotations\"] for token_id in annotation[\"quote\"][\"tokenIds\"]]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# I assume that speakers do not overlap\n", + "assert len(speaker_token_ids) == len(set(speaker_token_ids)), \"Speakers overlap\"\n", + "\n", + "# I assume that quotes do not overlap\n", + "assert len(quote_token_ids) == len(set(quote_token_ids)), \"Quotes overlap\"\n", + "\n", + "# I assume that speakers and quotes do not overlap\n", + "assert len(set(speaker_token_ids).intersection(set(quote_token_ids))) == 0, \"Speakers and quotes overlap\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "annotated_tokens = {\n", + " token_id: \"speaker\" for token_id in speaker_token_ids\n", + "}\n", + "\n", + "annotated_tokens.update({\n", + " token_id: \"quote\" for token_id in quote_token_ids\n", + "})\n", + "\n", + "annotated_tokens_fine = {k: v for k, v in annotated_tokens.items()}\n", + "annotated_tokens_fine.update({\n", + " token_id: f\"quote-{token_type.lower()}\" for token_id, token_type in zip(quote_token_ids, quote_types)\n", + "})" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tags = []\n", + "tags_fine = []\n", + "for token_id in document_token_ids:\n", + " if token_id in annotated_tokens:\n", + " tags.append(annotated_tokens[token_id])\n", + " tags_fine.append(annotated_tokens_fine[token_id])\n", + " else:\n", + " tags.append(\"O\")\n", + " tags_fine.append(\"O\") " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tags_fine" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "len(tags)" + ] + }, + { + "cell_type": "code", + "execution_count": 85, + "metadata": {}, + "outputs": [], + "source": [ + "coarse_label2lid = {\n", + " \"O\": 0,\n", + " \"speaker\": 1,\n", + " \"quote\": 2\n", + "}\n", + "coarse_id2label = {v: k for k, v in coarse_label2lid.items()}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = {\n", + " \"tokens\": document_tokens,\n", + " \"tags\": tags,\n", + " \"tags_fine\": tags_fine,\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.read_parquet(\"datasets/german-quotations/german_quotations_test.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
tokenstags
0[Bei, der, Frauen-Fußballweltmeisterschaft, in...[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...
1[Am, Freitag, haben, nach, Angaben, der, Deuts...[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...
2[Wie, die, Behörden, in, Singapur, mitteilten,...[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...
3[Im, Nordosten, Kenias, greift, seit, einem, M...[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...
4[Auf, der, Autobahn, A, 67, bei, Darmstadt, is...[O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ...
\n", + "
" + ], + "text/plain": [ + " tokens \\\n", + "0 [Bei, der, Frauen-Fußballweltmeisterschaft, in... \n", + "1 [Am, Freitag, haben, nach, Angaben, der, Deuts... \n", + "2 [Wie, die, Behörden, in, Singapur, mitteilten,... \n", + "3 [Im, Nordosten, Kenias, greift, seit, einem, M... \n", + "4 [Auf, der, Autobahn, A, 67, bei, Darmstadt, is... \n", + "\n", + " tags \n", + "0 [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ... \n", + "1 [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ... \n", + "2 [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ... \n", + "3 [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ... \n", + "4 [O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, ... " + ] + }, + "execution_count": 71, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": {}, + "outputs": [], + "source": [ + "df[\"isempty\"] = df[\"tags\"].apply(lambda x: len(set(x)) == 1 and \"O\" in x)" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "627" + ] + }, + "execution_count": 73, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[\"isempty\"].sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "998" + ] + }, + "execution_count": 74, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(df)" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "371" + ] + }, + "execution_count": 78, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# filter out empty documents\n", + "df2 = df[~df[\"isempty\"]]\n", + "len(df2)" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "627" + ] + }, + "execution_count": 79, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# filter out non empty documents\n", + "df3 = df[df[\"isempty\"]]\n", + "len(df3)" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "metadata": {}, + "outputs": [], + "source": [ + "# add 10% of the empty documents to the test set\n", + "df4 = df3.sample(frac=0.1, random_state=42)\n", + "df5 = pd.concat([df2, df4])" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "434" + ] + }, + "execution_count": 81, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(df5)" + ] + }, + { + "cell_type": "code", + "execution_count": 86, + "metadata": {}, + "outputs": [], + "source": [ + "df5[\"tags\"] = df5[\"tags\"].apply(lambda x: [coarse_label2lid[tag] for tag in x])" + ] + }, + { + "cell_type": "code", + "execution_count": 87, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
tokenstagsisempty
5[Großbritannien, friert, seine, Beziehungen, z...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...False
8[Der, Göttinger, Hotelmarketing-Experte, Chris...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...False
9[Am, Dienstagvormittag, wurde, ein, 44-jährige...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...False
12[Am, Dienstagmittag, sind, bei, einer, Massenk...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...False
13[Die, Untersuchung, der, in, der, Türkei, vere...[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...False
\n", + "
" + ], + "text/plain": [ + " tokens \\\n", + "5 [Großbritannien, friert, seine, Beziehungen, z... \n", + "8 [Der, Göttinger, Hotelmarketing-Experte, Chris... \n", + "9 [Am, Dienstagvormittag, wurde, ein, 44-jährige... \n", + "12 [Am, Dienstagmittag, sind, bei, einer, Massenk... \n", + "13 [Die, Untersuchung, der, in, der, Türkei, vere... \n", + "\n", + " tags isempty \n", + "5 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... False \n", + "8 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... False \n", + "9 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... False \n", + "12 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... False \n", + "13 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... False " + ] + }, + "execution_count": 87, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df5.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 88, + "metadata": {}, + "outputs": [], + "source": [ + "df5.to_parquet(\"datasets/german-quotations/german_direct_quotations.parquet\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/benchmarks/datasets/germanquad/.gitignore b/benchmarks/datasets/germanquad/.gitignore new file mode 100644 index 000000000..1a655b066 --- /dev/null +++ b/benchmarks/datasets/germanquad/.gitignore @@ -0,0 +1,4 @@ +* +!.gitignore +!README.md +!germanquad_dataset_creation.py diff --git a/benchmarks/datasets/germanquad/README.md b/benchmarks/datasets/germanquad/README.md new file mode 100644 index 000000000..670d53328 --- /dev/null +++ b/benchmarks/datasets/germanquad/README.md @@ -0,0 +1,42 @@ +# GermanQuAD (Benchmark Dataset) + +## What Is This Dataset About? + +GermanQuAD is a German extractive question answering dataset. +Each sample contains a context paragraph, a question, and one or more answer spans. + +## Where Can It Be Found? + +- Hugging Face dataset: https://huggingface.co/datasets/deepset/germanquad +- Project page: https://www.deepset.ai/germanquad + +## Benchmark Task Usage + +- Task: Extractive QA + +## How We Preprocess It + +Preprocessing is implemented in `germanquad_dataset_creation.py`. + +Main steps: + +1. Load split (default: `test`) from Hugging Face. +2. Keep `context`, `question`, and metadata (`id`, `title`). +3. Build a SQuAD-style reference object per sample: + - `id` + - `answers.text` + - `answers.answer_start` +4. Store this reference object as JSON string in the `reference` column. +5. Save to parquet (`test.parquet`). + +## Final Dataset Structure + +### File: `test.parquet` + +- `id`: sample id (string) +- `title`: article title +- `context`: context paragraph +- `question`: question text +- `answer_count`: number of annotated answers +- `is_answerable`: whether at least one answer span exists +- `reference`: JSON string with SQuAD-style reference payload diff --git a/benchmarks/datasets/germanquad/germanquad_dataset_creation.py b/benchmarks/datasets/germanquad/germanquad_dataset_creation.py new file mode 100644 index 000000000..ed4a25638 --- /dev/null +++ b/benchmarks/datasets/germanquad/germanquad_dataset_creation.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +import pandas as pd +from datasets import load_dataset + + +def _normalize_answers(raw_answers: Any) -> tuple[list[str], list[int]]: + if not isinstance(raw_answers, dict): + return [], [] + + answer_texts = [str(item) for item in raw_answers.get("text", [])] + answer_starts = [int(item) for item in raw_answers.get("answer_start", [])] + return answer_texts, answer_starts + + +def create_germanquad_dataset(split: str, output_path: Path) -> None: + dataset = load_dataset("deepset/germanquad", split=split) + + rows: list[dict[str, Any]] = [] + for index, sample in enumerate(dataset): + sample_id = str(sample.get("id") or index) + answer_texts, answer_starts = _normalize_answers(sample.get("answers")) + + reference_payload = { + "id": sample_id, + "answers": { + "text": answer_texts, + "answer_start": answer_starts, + }, + } + + rows.append( + { + "id": sample_id, + "title": str(sample.get("title") or ""), + "context": str(sample.get("context") or ""), + "question": str(sample.get("question") or ""), + "answer_count": len(answer_texts), + "is_answerable": len(answer_texts) > 0, + "reference": json.dumps(reference_payload, ensure_ascii=False), + } + ) + + df = pd.DataFrame(rows) + output_path.parent.mkdir(parents=True, exist_ok=True) + df.to_parquet(output_path, index=False) + + print("GermanQuAD dataset creation completed.") + print(f"Rows: {len(df)} -> {output_path}") + print(f"Answerable rows: {int(df['is_answerable'].sum())} / {len(df)}") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Download and preprocess GermanQuAD dataset for extractive QA benchmarks" + ) + parser.add_argument("--split", default="test", help="HuggingFace split") + parser.add_argument( + "--output", + default="datasets/germanquad/test.parquet", + help="Output parquet path relative to project root", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + project_root = Path(__file__).resolve().parents[2] + output_path = (project_root / args.output).resolve() + create_germanquad_dataset(split=args.split, output_path=output_path) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/datasets/imdb/.gitignore b/benchmarks/datasets/imdb/.gitignore new file mode 100644 index 000000000..659ba0542 --- /dev/null +++ b/benchmarks/datasets/imdb/.gitignore @@ -0,0 +1,4 @@ +* +!.gitignore +!README.md +!imdb_dataset_creation.ipynb diff --git a/benchmarks/datasets/imdb/README.md b/benchmarks/datasets/imdb/README.md new file mode 100644 index 000000000..4a3dd0520 --- /dev/null +++ b/benchmarks/datasets/imdb/README.md @@ -0,0 +1,95 @@ +# IMDB Genres (Benchmark Dataset) + +## What Is This Dataset About? + +This dataset contains movie descriptions and genre labels for document classification. + +It is used as a multi-label/multi-class style topic categorization dataset in the benchmark context. + +## Where Can It Be Found? + +- Hugging Face dataset: + - https://huggingface.co/datasets/jquigl/imdb-genres + +## Links (Website / Download / Citation) + +- Dataset card: + - https://huggingface.co/datasets/jquigl/imdb-genres + +## Benchmark Task Usage + +- Task 1: Document Classification + +## Dataset Size (Current Files) + +- `imdb_cleaned.parquet` (main benchmark file): 29756 samples + +## How We Preprocess It + +Preprocessing is implemented in `imdb_dataset_creation.ipynb`. + +Main steps: + +1. Load source data from Hugging Face export. +2. Inspect and normalize genre-related columns. +3. Remove the `Adult` genre from `expanded-genres` for cleaner label space. +4. Save processed output to `imdb_cleaned.parquet`. + +## Final Dataset Structure + +### File: `imdb_cleaned.parquet` + +- `movie title - year` +- `genre` +- `expanded-genres` +- `rating` +- `description` + +## Label Space (Most Important) + +### `genre` values (16) + +- `Action` +- `Adventure` +- `Animation` +- `Biography` +- `Crime` +- `Family` +- `Fantasy` +- `Film-noir` +- `History` +- `Horror` +- `Mystery` +- `Romance` +- `Scifi` +- `Sports` +- `Thriller` +- `War` + +### `expanded-genres` values after cleaning (25) + +- `Action` +- `Adventure` +- `Animation` +- `Biography` +- `Comedy` +- `Crime` +- `Drama` +- `Family` +- `Fantasy` +- `Film-Noir` +- `Game-Show` +- `History` +- `Horror` +- `Music` +- `Musical` +- `Mystery` +- `News` +- `Reality-TV` +- `Romance` +- `Sci-Fi` +- `Sport` +- `Talk-Show` +- `Thriller` +- `War` +- `Western` diff --git a/benchmarks/datasets/imdb/imdb_dataset_creation.ipynb b/benchmarks/datasets/imdb/imdb_dataset_creation.ipynb new file mode 100644 index 000000000..7b4f6e681 --- /dev/null +++ b/benchmarks/datasets/imdb/imdb_dataset_creation.ipynb @@ -0,0 +1,331 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "splits = {'train': 'train.csv', 'validation': 'validation.csv', 'test': 'test.csv'}\n", + "df = pd.read_csv(\"hf://datasets/jquigl/imdb-genres/\" + splits[\"test\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
movie title - yeargenreexpanded-genresratingdescription
0Son of the Wolf - nanAdventureAdventureNaNSet in 1800'2 Yukon, The Malamute Kid takes on...
1Firstborn - 2003ActionAction, Adventure, Fantasy6.1Sorcerers fight against themselves for ultimat...
213 Cameras - 2015ThrillerCrime, Drama, Horror5.2A newlywed couple, move into a new house acros...
3Straight Up, Now Tell Me... - nanRomanceRomanceNaNWhen a gay man brings his fiancee home to meet...
4The Ugly Duckling - 1959CrimeComedy, Crime, Sci-Fi5.5Henry Jeckle was always the outsider, a bungli...
\n", + "
" + ], + "text/plain": [ + " movie title - year genre expanded-genres \\\n", + "0 Son of the Wolf - nan Adventure Adventure \n", + "1 Firstborn - 2003 Action Action, Adventure, Fantasy \n", + "2 13 Cameras - 2015 Thriller Crime, Drama, Horror \n", + "3 Straight Up, Now Tell Me... - nan Romance Romance \n", + "4 The Ugly Duckling - 1959 Crime Comedy, Crime, Sci-Fi \n", + "\n", + " rating description \n", + "0 NaN Set in 1800'2 Yukon, The Malamute Kid takes on... \n", + "1 6.1 Sorcerers fight against themselves for ultimat... \n", + "2 5.2 A newlywed couple, move into a new house acros... \n", + "3 NaN When a gay man brings his fiancee home to meet... \n", + "4 5.5 Henry Jeckle was always the outsider, a bungli... " + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['Adventure', 'Action', 'Thriller', 'Romance', 'Crime', 'Fantasy',\n", + " 'Mystery', 'Horror', 'War', 'Family', 'Animation', 'Scifi',\n", + " 'Sports', 'History', 'Biography', 'Film-noir'], dtype=object)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[\"genre\"].unique()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "genre\n", + "Action 1336\n", + "Adventure 673\n", + "Animation 228\n", + "Biography 230\n", + "Crime 990\n", + "Family 434\n", + "Fantasy 463\n", + "Film-noir 32\n", + "History 240\n", + "Horror 990\n", + "Mystery 585\n", + "Romance 1392\n", + "Scifi 470\n", + "Sports 135\n", + "Thriller 1534\n", + "War 268\n", + "dtype: int64" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.sample(10000).groupby(\"genre\").size()" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "def filter_adult(labels):\n", + " return \", \".join([label for label in labels.split(\", \") if label != \"Adult\"])\n", + "\n", + "\n", + "df[\"expanded-genres\"] = df[\"expanded-genres\"].apply(filter_adult)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0 Adventure\n", + "1 Action, Adventure, Fantasy\n", + "2 Crime, Drama, Horror\n", + "3 Romance\n", + "4 Comedy, Crime, Sci-Fi\n", + " ... \n", + "29751 Crime, Drama, Thriller\n", + "29752 Adventure, Comedy\n", + "29753 Action, Comedy\n", + "29754 Comedy, Romance\n", + "29755 Action, Adventure, Crime\n", + "Name: expanded-genres, Length: 29756, dtype: object" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df[\"expanded-genres\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "labels = df[\"expanded-genres\"].to_list()" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "unique_labels = list(set(label for labels in labels for label in labels.split(\", \")))" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "unique_labels.sort()" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['Action',\n", + " 'Adventure',\n", + " 'Animation',\n", + " 'Biography',\n", + " 'Comedy',\n", + " 'Crime',\n", + " 'Drama',\n", + " 'Family',\n", + " 'Fantasy',\n", + " 'Film-Noir',\n", + " 'Game-Show',\n", + " 'History',\n", + " 'Horror',\n", + " 'Music',\n", + " 'Musical',\n", + " 'Mystery',\n", + " 'News',\n", + " 'Reality-TV',\n", + " 'Romance',\n", + " 'Sci-Fi',\n", + " 'Sport',\n", + " 'Talk-Show',\n", + " 'Thriller',\n", + " 'War',\n", + " 'Western']" + ] + }, + "execution_count": 26, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "unique_labels" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "# store the cleaned data in a new parquet file\n", + "df.to_parquet(\"imdb_cleaned.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/benchmarks/datasets/muc/.gitignore b/benchmarks/datasets/muc/.gitignore new file mode 100644 index 000000000..9a2aea9b2 --- /dev/null +++ b/benchmarks/datasets/muc/.gitignore @@ -0,0 +1,4 @@ +* +!.gitignore +!README.md +!muc_dataset_creation.ipynb diff --git a/benchmarks/datasets/muc/README.md b/benchmarks/datasets/muc/README.md new file mode 100644 index 000000000..97e153ffd --- /dev/null +++ b/benchmarks/datasets/muc/README.md @@ -0,0 +1,82 @@ +# MUC (Template Filling Benchmark Dataset) + +## What Is This Dataset About? + +This dataset contains incident reports with event templates (incident type plus entity slots). + +In this benchmark, it is used for information extraction and structured template filling. + +## Where Can It Be Found? + +- Local source files in this repository: + - `train.jsonl` + - `dev.jsonl` + - `test.jsonl` + +## Links (Website / Download / Citation) + +- MUC data index: + - https://www-nlpir.nist.gov/related_projects/muc/muc_data/muc_data_index.html + +## Benchmark Task Usage + +- Task 2.2: Template Filling + +## Dataset Size (Current Files) + +- `muc.parquet` (main benchmark file): 695 samples + +## How We Preprocess It + +Preprocessing is implemented in `muc_dataset_creation.ipynb`. + +Main steps: + +1. Load and merge `train.jsonl`, `dev.jsonl`, and `test.jsonl`. +2. Keep all records with exactly one template and sample a subset of zero-template records. +3. Flatten nested template fields into explicit columns. +4. Normalize incident label variants (for example, `attack / bombing` and `bombing / attack`). +5. Rename fields to benchmark schema names. +6. Keep slot values in list-based form where applicable. +7. Save processed output to `muc.parquet`. + +## Final Dataset Structure + +### File: `muc.parquet` + +- `docid`: document identifier +- `doctext`: source document text +- `incident`: incident type label (list form) +- `perpetrator`: perpetrator mentions +- `group perpetrator`: group perpetrator mentions +- `target`: target mentions +- `victim`: victim mentions +- `weapon`: weapon mentions + +### Incident Labels In `muc.parquet` + +Observed incident values: + +- `arson` +- `attack` +- `bombing` +- `kidnapping` +- `none` +- `robbery` + +## Source JSONL Record Structure (Raw) + +Each JSONL line contains: + +- `docid` +- `doctext` +- `templates` + +Template objects contain slot fields such as: + +- `incident_type` +- `PerpInd` +- `PerpOrg` +- `Target` +- `Victim` +- `Weapon` diff --git a/benchmarks/datasets/muc/muc_dataset_creation.ipynb b/benchmarks/datasets/muc/muc_dataset_creation.ipynb new file mode 100644 index 000000000..8d14bb8ae --- /dev/null +++ b/benchmarks/datasets/muc/muc_dataset_creation.ipynb @@ -0,0 +1,751 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "import json\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "paths = [\n", + " Path(\"datasets/muc/test.jsonl\"),\n", + " Path(\"datasets/muc/train.jsonl\"),\n", + " Path(\"datasets/muc/dev.jsonl\"),\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# read jsonl files\n", + "data = []\n", + "for file_path in paths:\n", + " with open(file_path, \"r\") as f:\n", + " for line in f:\n", + " data.append(json.loads(line))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1700" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(data)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "620" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# find all data where the templates array is len == 1\n", + "data_1 = [d for d in data if len(d[\"templates\"]) == 1]\n", + "len(data_1)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "758" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# find all data where the templates array is len == 0\n", + "data_2 = [d for d in data if len(d[\"templates\"]) == 0]\n", + "len(data_2)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "322" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# data where the templates array is len > 1\n", + "len(data) - len(data_1) - len(data_2)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "695" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# we want a dataset that consists of documents with exactly one template\n", + "# and we want to include 10% of the documents with no templates\n", + "# so we need to sample 10% of data_2\n", + "import random\n", + "\n", + "random.seed(42)\n", + "data_2_sample = random.sample(data_2, int(len(data_2) * 0.1))\n", + "\n", + "# combine data_1 and data_2_sample\n", + "data_combined = data_1 + data_2_sample\n", + "len(data_combined)" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'docid': 'TST3-MUC4-0002',\n", + " 'doctext': 'those accused of the assassination of six jesuits will have a \"fair trial\" and if found guilty, will be punished whether they are civilians, military, or influential people, supreme court president dr mauricio gutierrez castro said. the technical investigation commission has determined that some military were reportedly involved in the assassination of the six jesuits and their two maids, which took place at daybreak on 16 november, as reported by president alfredo cristiani on 7 january. \"the local and international community can rest assured the salvadoran judiciary system will not hesitate to enforce the law upon the authors of this horrible crime,\" dr gutierrez pointed out. gutierrez said he does not know how many people are involved or their military ranks, because the commission to investigate criminal actions is still conducting investigations and has not presented its report to the tribunal. however, general prosecutor dr mauricio eduardo colorado maintained that the military suspects \"are regrouped and have been put in custody.\" however, he did not reveal the names or the military ranks of the suspects. the prosecutor said that the scientific test conducted by the commission to investigate criminal actions and the specialized police working on this case have determined that the crime could have been perpetrated by armed forces members. \"the attorney general office will proceed according to the law and against whoever turns out to be guilty, because it is the constitutional duty of the attorney general\\'s office,\" colorado said. asked about this issue, justice minister dr oscar alfredo santamaria, president of the commission to investigate criminal actions said: \"we understand that the most recent information on this case was announced by the president of the republic a few days ago.\" he refused to make any further statements. new uca (central american university) rector jesuit francisco estrada admitted president alfredo cristiani has taken \"a step forward\" by unmasking the suspects in this crime, \"because his ethic principles do not tolerate such atrocities.\" estrada said he understands why cristiani, during his 7 january speech, did not give any names, \"because the investigations are still being conducted.\" moreover, there must not only be one suspect, but rather several.\" before meeting with the reporters, the uca rector met with officials from scotland yard. a few days ago, he also met spanish, canadian, and u.s. policemen who, at the government\\'s request, are assisting the salvadoran commission in charge of clarifying this case.',\n", + " 'templates': [{'incident_type': 'attack',\n", + " 'PerpInd': [[['military', 141],\n", + " ['some military', 295],\n", + " ['military suspects', 1002],\n", + " ['armed forces members', 1360]]],\n", + " 'PerpOrg': [[['armed forces', 1360]]],\n", + " 'Target': [],\n", + " 'Victim': [],\n", + " 'Weapon': []}]}" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# create a dataframe\n", + "data_combined[1]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "for data in data_combined:\n", + " if len(data[\"templates\"]) == 1:\n", + " # flatten the lists, only consider the text, not the int offset\n", + " template = data[\"templates\"][0]\n", + " for key, value in template.items():\n", + " if type(value) == str:\n", + " continue\n", + " \n", + " texts = []\n", + " for group in value:\n", + " for item in group:\n", + " texts.append(item[0])\n", + " template[key] = texts\n", + " data.update(data[\"templates\"][0])\n", + " else:\n", + " data.update({\n", + " \"incident_type\": \"none\",\n", + " \"PerpInd\": [],\n", + " \"PerpOrg\": [],\n", + " \"Target\": [],\n", + " \"Victim\": [],\n", + " \"Weapon\": [],\n", + " })\n", + " del data[\"templates\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'docid': 'TST3-MUC4-0019',\n", + " 'doctext': 'oil will not be pumped through the cano limon-covenas pipeline again until 30 may, because it was again blown up today near urum municipality, norte de santander department.',\n", + " 'incident_type': 'bombing',\n", + " 'PerpInd': [],\n", + " 'PerpOrg': [],\n", + " 'Target': ['cano limon-covenas pipeline', 'pipeline'],\n", + " 'Victim': [],\n", + " 'Weapon': []}" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data_combined[5]" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.DataFrame(data_combined)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['attack', 'kidnapping', 'bombing', 'attack / bombing', 'robbery',\n", + " 'arson', 'bombing / attack', 'none'], dtype=object)" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "unique_incident_types = df[\"incident_type\"].unique()\n", + "unique_incident_types" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "# find all data where the incident_type is \"attack / bombing\"\n", + "df[df[\"incident_type\"] == \"attack / bombing\"] = df[df[\"incident_type\"] == \"attack / bombing\"].replace(\"attack / bombing\", \"attack\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "# find all data where the incident_type is \"bombing / attack\"\n", + "df[df[\"incident_type\"] == \"bombing / attack\"] = df[df[\"incident_type\"] == \"bombing / attack\"].replace(\"bombing / attack\", \"bombing\")" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dociddoctextincident_typePerpIndPerpOrgTargetVictimWeapon
0TST3-MUC4-0001we are not demanding that they stop their oper...attack[][][][][]
1TST3-MUC4-0002those accused of the assassination of six jesu...attack[military, some military, military suspects, a...[armed forces][][][]
2TST3-MUC4-0003the national police reported today that over 1...attack[members of the maoist terrorist organization ...[shining path][][enrique lopez albujar trint][]
3TST3-MUC4-0005salvadoran social democratic politician hector...kidnapping[heavily armed men][][][hector oqueli colindres, gilda flores][]
4TST3-MUC4-0011the dissemination of a document questioning co...kidnapping[members of the manuel gustavo chacon sovereig...[eln, army of national liberation][][][]
\n", + "
" + ], + "text/plain": [ + " docid doctext \\\n", + "0 TST3-MUC4-0001 we are not demanding that they stop their oper... \n", + "1 TST3-MUC4-0002 those accused of the assassination of six jesu... \n", + "2 TST3-MUC4-0003 the national police reported today that over 1... \n", + "3 TST3-MUC4-0005 salvadoran social democratic politician hector... \n", + "4 TST3-MUC4-0011 the dissemination of a document questioning co... \n", + "\n", + " incident_type PerpInd \\\n", + "0 attack [] \n", + "1 attack [military, some military, military suspects, a... \n", + "2 attack [members of the maoist terrorist organization ... \n", + "3 kidnapping [heavily armed men] \n", + "4 kidnapping [members of the manuel gustavo chacon sovereig... \n", + "\n", + " PerpOrg Target \\\n", + "0 [] [] \n", + "1 [armed forces] [] \n", + "2 [shining path] [] \n", + "3 [] [] \n", + "4 [eln, army of national liberation] [] \n", + "\n", + " Victim Weapon \n", + "0 [] [] \n", + "1 [] [] \n", + "2 [enrique lopez albujar trint] [] \n", + "3 [hector oqueli colindres, gilda flores] [] \n", + "4 [] [] " + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dociddoctextincidentperpetratorgroup perpetratortargetvictimweapon
0TST3-MUC4-0001we are not demanding that they stop their oper...attack[][][][][]
1TST3-MUC4-0002those accused of the assassination of six jesu...attack[military, some military, military suspects, a...[armed forces][][][]
2TST3-MUC4-0003the national police reported today that over 1...attack[members of the maoist terrorist organization ...[shining path][][enrique lopez albujar trint][]
3TST3-MUC4-0005salvadoran social democratic politician hector...kidnapping[heavily armed men][][][hector oqueli colindres, gilda flores][]
4TST3-MUC4-0011the dissemination of a document questioning co...kidnapping[members of the manuel gustavo chacon sovereig...[eln, army of national liberation][][][]
\n", + "
" + ], + "text/plain": [ + " docid doctext \\\n", + "0 TST3-MUC4-0001 we are not demanding that they stop their oper... \n", + "1 TST3-MUC4-0002 those accused of the assassination of six jesu... \n", + "2 TST3-MUC4-0003 the national police reported today that over 1... \n", + "3 TST3-MUC4-0005 salvadoran social democratic politician hector... \n", + "4 TST3-MUC4-0011 the dissemination of a document questioning co... \n", + "\n", + " incident perpetrator \\\n", + "0 attack [] \n", + "1 attack [military, some military, military suspects, a... \n", + "2 attack [members of the maoist terrorist organization ... \n", + "3 kidnapping [heavily armed men] \n", + "4 kidnapping [members of the manuel gustavo chacon sovereig... \n", + "\n", + " group perpetrator target \\\n", + "0 [] [] \n", + "1 [armed forces] [] \n", + "2 [shining path] [] \n", + "3 [] [] \n", + "4 [eln, army of national liberation] [] \n", + "\n", + " victim weapon \n", + "0 [] [] \n", + "1 [] [] \n", + "2 [enrique lopez albujar trint] [] \n", + "3 [hector oqueli colindres, gilda flores] [] \n", + "4 [] [] " + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# column renaming\n", + "df = df.rename(columns={\n", + " \"incident_type\": \"incident\",\n", + " \"PerpInd\": \"perpetrator\",\n", + " \"PerpOrg\": \"group perpetrator\",\n", + " \"Target\": \"target\",\n", + " \"Victim\": \"victim\",\n", + " \"Weapon\": \"weapon\"})\n", + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "# make incident to a list\n", + "df[\"incident\"] = df[\"incident\"].apply(lambda x: [x])" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dociddoctextincidentperpetratorgroup perpetratortargetvictimweapon
0TST3-MUC4-0001we are not demanding that they stop their oper...[attack][][][][][]
1TST3-MUC4-0002those accused of the assassination of six jesu...[attack][military, some military, military suspects, a...[armed forces][][][]
2TST3-MUC4-0003the national police reported today that over 1...[attack][members of the maoist terrorist organization ...[shining path][][enrique lopez albujar trint][]
3TST3-MUC4-0005salvadoran social democratic politician hector...[kidnapping][heavily armed men][][][hector oqueli colindres, gilda flores][]
4TST3-MUC4-0011the dissemination of a document questioning co...[kidnapping][members of the manuel gustavo chacon sovereig...[eln, army of national liberation][][][]
\n", + "
" + ], + "text/plain": [ + " docid doctext \\\n", + "0 TST3-MUC4-0001 we are not demanding that they stop their oper... \n", + "1 TST3-MUC4-0002 those accused of the assassination of six jesu... \n", + "2 TST3-MUC4-0003 the national police reported today that over 1... \n", + "3 TST3-MUC4-0005 salvadoran social democratic politician hector... \n", + "4 TST3-MUC4-0011 the dissemination of a document questioning co... \n", + "\n", + " incident perpetrator \\\n", + "0 [attack] [] \n", + "1 [attack] [military, some military, military suspects, a... \n", + "2 [attack] [members of the maoist terrorist organization ... \n", + "3 [kidnapping] [heavily armed men] \n", + "4 [kidnapping] [members of the manuel gustavo chacon sovereig... \n", + "\n", + " group perpetrator target \\\n", + "0 [] [] \n", + "1 [armed forces] [] \n", + "2 [shining path] [] \n", + "3 [] [] \n", + "4 [eln, army of national liberation] [] \n", + "\n", + " victim weapon \n", + "0 [] [] \n", + "1 [] [] \n", + "2 [enrique lopez albujar trint] [] \n", + "3 [hector oqueli colindres, gilda flores] [] \n", + "4 [] [] " + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "df.to_parquet(\"muc.parquet\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/benchmarks/datasets/pubmed200k/.gitignore b/benchmarks/datasets/pubmed200k/.gitignore new file mode 100644 index 000000000..94f335824 --- /dev/null +++ b/benchmarks/datasets/pubmed200k/.gitignore @@ -0,0 +1,3 @@ +*.parquet +*.txt +*.json diff --git a/benchmarks/datasets/pubmed200k/README.md b/benchmarks/datasets/pubmed200k/README.md new file mode 100644 index 000000000..b8d74534f --- /dev/null +++ b/benchmarks/datasets/pubmed200k/README.md @@ -0,0 +1,35 @@ +# PubMed 200k RCT + +a large dataset for sequential sentence classification + +Paper: https://aclanthology.org/I17-2052.pdf +Download: https://github.com/Franck-Dernoncourt/pubmed-rct + +## Setup +Run pubmed200k.iypnb to preprocess the dataset. + +## What is PubMed 200k RCT + +``` +PubMed 200k RCT is new dataset based on PubMed for sequential sentence classification. + The dataset consists of approximately 200,000 abstracts of randomized controlled trials, totaling 2.3 million sentences. + Each sentence of each abstract is labeled with their role in the abstract using one of the following classes: background, objective, method, result, or conclusion. +``` + +## Labels + +The label definitions are written by us. +Their paper does not provide any further definitions of the labels. +We use the same definitions in pubmed200k and csabstruct. + +``` +label_dict = { + "background": "Provides context or previous knowledge relevant to the research topic. Think of it as setting the stage for the study.", + "methods": "Describes the procedures and techniques used in the research. This includes the study design, data collection, and analysis methods.", + "objective": "States the main goal or purpose of the research. What question is this work trying to answer?", + "results": "Presents the findings or outcomes of the research. This often includes statistical data, tables, and figures.", + "conclusions": "Summarizes the key findings of the research and draw inferences from those findings. They provide closure to the abstract, summarizing the overall contribution of the research." +} +``` + +{'results', 'objective', 'background', 'methods', 'conclusions'} diff --git a/benchmarks/datasets/pubmed200k/pubmed200k.ipynb b/benchmarks/datasets/pubmed200k/pubmed200k.ipynb new file mode 100644 index 000000000..2a9dbbede --- /dev/null +++ b/benchmarks/datasets/pubmed200k/pubmed200k.ipynb @@ -0,0 +1,253 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "from typing import List\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "datasets_path = Path(\"./datasets/pubmed200k\")\n", + "valid_path = datasets_path / \"dev.txt\"\n", + "test_path = datasets_path / \"test.txt\"\n", + "train_path = datasets_path / \"train.txt\"" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [], + "source": [ + "def create_dataset(path: Path):\n", + " # extract sentences and labels\n", + " sentences_list: List[List[str]] = []\n", + " labels_list: List[List[str]] = []\n", + " with path.open('r') as file:\n", + " lines = file.readlines()\n", + "\n", + " sentences: List[str] = []\n", + " labels: List[str] = []\n", + "\n", + " for line in lines:\n", + " if line == \"\\n\":\n", + " sentences_list.append(sentences)\n", + " labels_list.append(labels)\n", + " sentences = []\n", + " labels = []\n", + " \n", + " splitted = line.strip().split(\"\\t\")\n", + " if(len(splitted) == 2):\n", + " labels.append(splitted[0].lower())\n", + " sentences.append(splitted[1])\n", + "\n", + " # create dataframe\n", + " df = pd.DataFrame({\"sentences\": sentences_list, \"labels\": labels_list})\n", + "\n", + " # save dataframe\n", + " df.to_parquet(path.with_suffix(\".parquet\"))\n", + "\n", + " # unique labels\n", + " unique_labels = set()\n", + " for labels in labels_list:\n", + " unique_labels.update(labels)\n", + " return unique_labels" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "l1 = create_dataset(valid_path)\n", + "l2 = create_dataset(test_path)\n", + "l3 = create_dataset(train_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'results', 'objective', 'background', 'methods', 'conclusions'}\n" + ] + } + ], + "source": [ + "all_labels = l1.union(l2).union(l3)\n", + "print(all_labels)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "# read datasets\n", + "valid_df = pd.read_parquet(valid_path.with_suffix(\".parquet\"))\n", + "test_df = pd.read_parquet(test_path.with_suffix(\".parquet\"))\n", + "train_df = pd.read_parquet(train_path.with_suffix(\".parquet\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
sentenceslabels
0[IgE sensitization to Aspergillus fumigatus an...[background, background, objective, methods, m...
1[Opioid antagonists ( e.g. , naltrexone ) and ...[background, background, background, objective...
2[The sequencing of learning materials greatly ...[background, background, background, objective...
3[Patient adherence to appointments is key to i...[background, background, background, methods, ...
4[Insufficient skills in drug dose calculations...[background, background, background, backgroun...
\n", + "
" + ], + "text/plain": [ + " sentences \\\n", + "0 [IgE sensitization to Aspergillus fumigatus an... \n", + "1 [Opioid antagonists ( e.g. , naltrexone ) and ... \n", + "2 [The sequencing of learning materials greatly ... \n", + "3 [Patient adherence to appointments is key to i... \n", + "4 [Insufficient skills in drug dose calculations... \n", + "\n", + " labels \n", + "0 [background, background, objective, methods, m... \n", + "1 [background, background, background, objective... \n", + "2 [background, background, background, objective... \n", + "3 [background, background, background, methods, ... \n", + "4 [background, background, background, backgroun... " + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "valid_df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0 [background, background, objective, methods, m...\n", + "1 [background, background, background, objective...\n", + "2 [background, background, background, objective...\n", + "3 [background, background, background, methods, ...\n", + "4 [background, background, background, backgroun...\n", + " ... \n", + "2495 [background, background, background, backgroun...\n", + "2496 [background, background, methods, methods, met...\n", + "2497 [background, background, methods, methods, met...\n", + "2498 [background, methods, methods, methods, method...\n", + "2499 [background, methods, methods, methods, method...\n", + "Name: labels, Length: 2500, dtype: object" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "valid_df[\"labels\"]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sent-class", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.15" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/benchmarks/datasets/squad/.gitignore b/benchmarks/datasets/squad/.gitignore new file mode 100644 index 000000000..54286284a --- /dev/null +++ b/benchmarks/datasets/squad/.gitignore @@ -0,0 +1,4 @@ +* +!.gitignore +!README.md +!squad_dataset_creation.py diff --git a/benchmarks/datasets/squad/README.md b/benchmarks/datasets/squad/README.md new file mode 100644 index 000000000..06741a9dc --- /dev/null +++ b/benchmarks/datasets/squad/README.md @@ -0,0 +1,42 @@ +# SQuAD (Benchmark Dataset) + +## What Is This Dataset About? + +SQuAD is an English extractive question answering dataset. +Each sample contains a context paragraph, a question, and one or more answer spans. + +## Where Can It Be Found? + +- Hugging Face dataset: https://huggingface.co/datasets/squad +- Original paper: https://arxiv.org/abs/1606.05250 + +## Benchmark Task Usage + +- Task: Extractive QA + +## How We Preprocess It + +Preprocessing is implemented in `squad_dataset_creation.py`. + +Main steps: + +1. Load split (default: `validation`) from Hugging Face. +2. Keep `context`, `question`, and metadata (`id`, `title`). +3. Build a SQuAD-style reference object per sample: + - `id` + - `answers.text` + - `answers.answer_start` +4. Store this reference object as JSON string in the `reference` column. +5. Save to parquet (`validation.parquet`). + +## Final Dataset Structure + +### File: `validation.parquet` + +- `id`: sample id (string) +- `title`: article title +- `context`: context paragraph +- `question`: question text +- `answer_count`: number of annotated answers +- `is_answerable`: whether at least one answer span exists +- `reference`: JSON string with SQuAD-style reference payload diff --git a/benchmarks/datasets/squad/squad_dataset_creation.py b/benchmarks/datasets/squad/squad_dataset_creation.py new file mode 100644 index 000000000..f00efa6a7 --- /dev/null +++ b/benchmarks/datasets/squad/squad_dataset_creation.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +import pandas as pd +from datasets import load_dataset + + +def _normalize_answers(raw_answers: Any) -> tuple[list[str], list[int]]: + if not isinstance(raw_answers, dict): + return [], [] + + answer_texts = [str(item) for item in raw_answers.get("text", [])] + answer_starts = [int(item) for item in raw_answers.get("answer_start", [])] + return answer_texts, answer_starts + + +def create_squad_dataset(split: str, output_path: Path) -> None: + dataset = load_dataset("squad", split=split) + + rows: list[dict[str, Any]] = [] + for index, sample in enumerate(dataset): + sample_id = str(sample.get("id") or index) + answer_texts, answer_starts = _normalize_answers(sample.get("answers")) + + reference_payload = { + "id": sample_id, + "answers": { + "text": answer_texts, + "answer_start": answer_starts, + }, + } + + rows.append( + { + "id": sample_id, + "title": str(sample.get("title") or ""), + "context": str(sample.get("context") or ""), + "question": str(sample.get("question") or ""), + "answer_count": len(answer_texts), + "is_answerable": len(answer_texts) > 0, + "reference": json.dumps(reference_payload, ensure_ascii=False), + } + ) + + df = pd.DataFrame(rows) + output_path.parent.mkdir(parents=True, exist_ok=True) + df.to_parquet(output_path, index=False) + + print("SQuAD dataset creation completed.") + print(f"Rows: {len(df)} -> {output_path}") + print(f"Answerable rows: {int(df['is_answerable'].sum())} / {len(df)}") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Download and preprocess SQuAD dataset for extractive QA benchmarks" + ) + parser.add_argument("--split", default="validation", help="HuggingFace split") + parser.add_argument( + "--output", + default="datasets/squad/validation.parquet", + help="Output parquet path relative to project root", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + project_root = Path(__file__).resolve().parents[2] + output_path = (project_root / args.output).resolve() + create_squad_dataset(split=args.split, output_path=output_path) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/datasets/squad2/.gitignore b/benchmarks/datasets/squad2/.gitignore new file mode 100644 index 000000000..ec2cd7db0 --- /dev/null +++ b/benchmarks/datasets/squad2/.gitignore @@ -0,0 +1,4 @@ +* +!.gitignore +!README.md +!squad2_dataset_creation.py diff --git a/benchmarks/datasets/squad2/README.md b/benchmarks/datasets/squad2/README.md new file mode 100644 index 000000000..8967fcbde --- /dev/null +++ b/benchmarks/datasets/squad2/README.md @@ -0,0 +1,43 @@ +# SQuAD v2 (Benchmark Dataset) + +## What Is This Dataset About? + +SQuAD v2 extends SQuAD by adding unanswerable questions. +Models must extract exact spans when answerable and abstain when not answerable. + +## Where Can It Be Found? + +- Hugging Face dataset: https://huggingface.co/datasets/squad_v2 +- Original paper: https://arxiv.org/abs/1806.03822 + +## Benchmark Task Usage + +- Task: Extractive QA + +## How We Preprocess It + +Preprocessing is implemented in `squad2_dataset_creation.py`. + +Main steps: + +1. Load split (default: `validation`) from Hugging Face. +2. Keep `context`, `question`, and metadata (`id`, `title`, `is_impossible`). +3. Build a SQuAD-style reference object per sample: + - `id` + - `answers.text` + - `answers.answer_start` +4. Store this reference object as JSON string in the `reference` column. +5. Save to parquet (`validation.parquet`). + +## Final Dataset Structure + +### File: `validation.parquet` + +- `id`: sample id (string) +- `title`: article title +- `context`: context paragraph +- `question`: question text +- `is_impossible`: original dataset flag +- `answer_count`: number of annotated answers +- `is_answerable`: whether at least one answer span exists +- `reference`: JSON string with SQuAD-style reference payload diff --git a/benchmarks/datasets/squad2/squad2_dataset_creation.py b/benchmarks/datasets/squad2/squad2_dataset_creation.py new file mode 100644 index 000000000..79888aa18 --- /dev/null +++ b/benchmarks/datasets/squad2/squad2_dataset_creation.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +import pandas as pd +from datasets import load_dataset + + +def _normalize_answers(raw_answers: Any) -> tuple[list[str], list[int]]: + if not isinstance(raw_answers, dict): + return [], [] + + answer_texts = [str(item) for item in raw_answers.get("text", [])] + answer_starts = [int(item) for item in raw_answers.get("answer_start", [])] + return answer_texts, answer_starts + + +def create_squad2_dataset(split: str, output_path: Path) -> None: + dataset = load_dataset("squad_v2", split=split) + + rows: list[dict[str, Any]] = [] + for index, sample in enumerate(dataset): + sample_id = str(sample.get("id") or index) + answer_texts, answer_starts = _normalize_answers(sample.get("answers")) + is_impossible = bool(sample.get("is_impossible", False)) + + reference_payload = { + "id": sample_id, + "answers": { + "text": answer_texts, + "answer_start": answer_starts, + }, + } + + rows.append( + { + "id": sample_id, + "title": str(sample.get("title") or ""), + "context": str(sample.get("context") or ""), + "question": str(sample.get("question") or ""), + "is_impossible": is_impossible, + "answer_count": len(answer_texts), + "is_answerable": len(answer_texts) > 0, + "reference": json.dumps(reference_payload, ensure_ascii=False), + } + ) + + df = pd.DataFrame(rows) + output_path.parent.mkdir(parents=True, exist_ok=True) + df.to_parquet(output_path, index=False) + + print("SQuAD v2 dataset creation completed.") + print(f"Rows: {len(df)} -> {output_path}") + print(f"Answerable rows: {int(df['is_answerable'].sum())} / {len(df)}") + print(f"Unanswerable rows: {int((~df['is_answerable']).sum())} / {len(df)}") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Download and preprocess SQuAD v2 dataset for extractive QA benchmarks" + ) + parser.add_argument("--split", default="validation", help="HuggingFace split") + parser.add_argument( + "--output", + default="datasets/squad2/validation.parquet", + help="Output parquet path relative to project root", + ) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + project_root = Path(__file__).resolve().parents[2] + output_path = (project_root / args.output).resolve() + create_squad2_dataset(split=args.split, output_path=output_path) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/datasets/tagesschau/.gitignore b/benchmarks/datasets/tagesschau/.gitignore new file mode 100644 index 000000000..f56604a8c --- /dev/null +++ b/benchmarks/datasets/tagesschau/.gitignore @@ -0,0 +1,4 @@ +* +!.gitignore +!README.md +!tagesschau_dataset_creation.ipynb diff --git a/benchmarks/datasets/tagesschau/README.md b/benchmarks/datasets/tagesschau/README.md new file mode 100644 index 000000000..189ade157 --- /dev/null +++ b/benchmarks/datasets/tagesschau/README.md @@ -0,0 +1,109 @@ +# Tagesschau 2018-2023 (Benchmark Dataset) + +## What Is This Dataset About? + +German news articles from Tagesschau, used for topic/document classification. + +The benchmark preprocessing derives hierarchical topic labels from article URLs. + +## Where Can It Be Found? + +- Hugging Face dataset: + - https://huggingface.co/datasets/bjoernp/tagesschau-2018-2023 + +## Links (Website / Download / Citation) + +- Dataset card: + - https://huggingface.co/datasets/bjoernp/tagesschau-2018-2023 + +## Benchmark Task Usage + +- Task 1: Document Classification + +## Dataset Size (Current Files) + +- `tagesschau_cleaned.parquet` (main benchmark file): 11473 samples + +## How We Preprocess It + +Preprocessing is implemented in `tagesschau_dataset_creation.ipynb`. + +Main steps: + +1. Load raw Parquet directly from Hugging Face storage. +2. Derive URL depth (`count`) from `link` and filter to expected structure. +3. Parse URL path segments into: + - `main_tag` + - `sub_tag` +4. Remove predefined noisy/unwanted tags and subtags. +5. Build a merged label field `tag`. +6. Export cleaned dataset to `tagesschau_cleaned.parquet`. + +## Final Dataset Structure + +### File: `tagesschau_cleaned.parquet` + +- `date` +- `headline` +- `short_headline` +- `short_text` +- `article` +- `link` +- `main_tag` +- `sub_tag` +- `tag` +- `__index_level_0__` (pandas index artifact) + +## Label Space (Most Important) + +### `main_tag` classes (4) + +- `ausland` (5800) +- `wirtschaft` (3202) +- `inland` (1935) +- `wissen` (536) + +### `sub_tag` classes (20) + +- `europa` (2860) +- `asien` (1370) +- `innenpolitik` (1256) +- `amerika` (1226) +- `unternehmen` (1160) +- `verbraucher` (667) +- `gesellschaft` (550) +- `weltwirtschaft` (502) +- `konjunktur` (307) +- `technologie` (302) +- `afrika` (282) +- `finanzen` (279) +- `klima` (241) +- `gesundheit` (144) +- `deutschlandtrend` (102) +- `forschung` (97) +- `ozeanien` (62) +- `boerse` (39) +- `mittendrin` (27) + +### `tag` classes (20) + +- `ausland/europa` (2860) +- `ausland/asien` (1370) +- `inland/innenpolitik` (1256) +- `ausland/amerika` (1226) +- `wirtschaft/unternehmen` (1160) +- `wirtschaft/verbraucher` (667) +- `inland/gesellschaft` (550) +- `wirtschaft/weltwirtschaft` (502) +- `wirtschaft/konjunktur` (307) +- `ausland/afrika` (282) +- `wirtschaft/finanzen` (279) +- `wirtschaft/technologie` (248) +- `wissen/klima` (241) +- `wissen/gesundheit` (144) +- `inland/deutschlandtrend` (102) +- `wissen/forschung` (97) +- `ausland/ozeanien` (62) +- `wissen/technologie` (54) +- `wirtschaft/boerse` (39) +- `inland/mittendrin` (27) diff --git a/benchmarks/datasets/tagesschau/tagesschau_dataset_creation.ipynb b/benchmarks/datasets/tagesschau/tagesschau_dataset_creation.ipynb new file mode 100644 index 000000000..25630cfba --- /dev/null +++ b/benchmarks/datasets/tagesschau/tagesschau_dataset_creation.ipynb @@ -0,0 +1,1931 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "df = pd.read_parquet(\"hf://datasets/bjoernp/tagesschau-2018-2023/data/train-00000-of-00001-1c2a165f0626c4a6.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dateheadlineshort_headlineshort_textarticlelink
02023-04-27Türkei-Wahl in Deutschland startet1,5 Millionen StimmberechtigteEtwa 1,5 Millionen türkische Staatsbürger in D...Etwa 1,5 Millionen türkische Staatsbürger in D.../ausland/europa/erdogan-wahlkampfpause-gesundh...
12023-04-27Bolsonaro bestreitet Verwicklung in Regierungs...BrasilienBrasiliens Ex-Präsident Bolsonaro hat eine Ver...Brasiliens Ex-Präsident Bolsonaro hat eine Ver.../ausland/amerika/bolsonaro-brasilien-sturm-reg...
22023-04-27Streiten, ob Hilfe wirklich richtig istAfghanistan-Politik der BundesregierungDeutschland hat seine Finanzhilfen für Afghani...Deutschland hat seine Finanzhilfen für Afghani.../inland/innenpolitik/afghanistan-deutschland-h...
32023-04-27Republikaner machen Druck auf BidenUS-HaushaltsstreitDie Republikaner im US-Kongress wollen Präside...Die Republikaner im US-Kongress wollen Präside.../ausland/amerika/usa-haushaltsstreit-biden-rep...
42023-04-27Russland plant Schein-Organisation zur Einflus...StrategiepapierRussland will offenbar das Thema Ostseeverschm...Russland will offenbar das Thema Ostseeverschm.../investigativ/ndr-wdr/russland-kreml-strategie...
\n", + "
" + ], + "text/plain": [ + " date headline \\\n", + "0 2023-04-27 Türkei-Wahl in Deutschland startet \n", + "1 2023-04-27 Bolsonaro bestreitet Verwicklung in Regierungs... \n", + "2 2023-04-27 Streiten, ob Hilfe wirklich richtig ist \n", + "3 2023-04-27 Republikaner machen Druck auf Biden \n", + "4 2023-04-27 Russland plant Schein-Organisation zur Einflus... \n", + "\n", + " short_headline \\\n", + "0 1,5 Millionen Stimmberechtigte \n", + "1 Brasilien \n", + "2 Afghanistan-Politik der Bundesregierung \n", + "3 US-Haushaltsstreit \n", + "4 Strategiepapier \n", + "\n", + " short_text \\\n", + "0 Etwa 1,5 Millionen türkische Staatsbürger in D... \n", + "1 Brasiliens Ex-Präsident Bolsonaro hat eine Ver... \n", + "2 Deutschland hat seine Finanzhilfen für Afghani... \n", + "3 Die Republikaner im US-Kongress wollen Präside... \n", + "4 Russland will offenbar das Thema Ostseeverschm... \n", + "\n", + " article \\\n", + "0 Etwa 1,5 Millionen türkische Staatsbürger in D... \n", + "1 Brasiliens Ex-Präsident Bolsonaro hat eine Ver... \n", + "2 Deutschland hat seine Finanzhilfen für Afghani... \n", + "3 Die Republikaner im US-Kongress wollen Präside... \n", + "4 Russland will offenbar das Thema Ostseeverschm... \n", + "\n", + " link \n", + "0 /ausland/europa/erdogan-wahlkampfpause-gesundh... \n", + "1 /ausland/amerika/bolsonaro-brasilien-sturm-reg... \n", + "2 /inland/innenpolitik/afghanistan-deutschland-h... \n", + "3 /ausland/amerika/usa-haushaltsstreit-biden-rep... \n", + "4 /investigativ/ndr-wdr/russland-kreml-strategie... " + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "df[\"count\"] = df[\"link\"].apply(lambda x: len(x.split(\"/\")))" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dateheadlineshort_headlineshort_textarticlelinkcount
02023-04-27Türkei-Wahl in Deutschland startet1,5 Millionen StimmberechtigteEtwa 1,5 Millionen türkische Staatsbürger in D...Etwa 1,5 Millionen türkische Staatsbürger in D.../ausland/europa/erdogan-wahlkampfpause-gesundh...4
12023-04-27Bolsonaro bestreitet Verwicklung in Regierungs...BrasilienBrasiliens Ex-Präsident Bolsonaro hat eine Ver...Brasiliens Ex-Präsident Bolsonaro hat eine Ver.../ausland/amerika/bolsonaro-brasilien-sturm-reg...4
22023-04-27Streiten, ob Hilfe wirklich richtig istAfghanistan-Politik der BundesregierungDeutschland hat seine Finanzhilfen für Afghani...Deutschland hat seine Finanzhilfen für Afghani.../inland/innenpolitik/afghanistan-deutschland-h...4
32023-04-27Republikaner machen Druck auf BidenUS-HaushaltsstreitDie Republikaner im US-Kongress wollen Präside...Die Republikaner im US-Kongress wollen Präside.../ausland/amerika/usa-haushaltsstreit-biden-rep...4
42023-04-27Russland plant Schein-Organisation zur Einflus...StrategiepapierRussland will offenbar das Thema Ostseeverschm...Russland will offenbar das Thema Ostseeverschm.../investigativ/ndr-wdr/russland-kreml-strategie...4
\n", + "
" + ], + "text/plain": [ + " date headline \\\n", + "0 2023-04-27 Türkei-Wahl in Deutschland startet \n", + "1 2023-04-27 Bolsonaro bestreitet Verwicklung in Regierungs... \n", + "2 2023-04-27 Streiten, ob Hilfe wirklich richtig ist \n", + "3 2023-04-27 Republikaner machen Druck auf Biden \n", + "4 2023-04-27 Russland plant Schein-Organisation zur Einflus... \n", + "\n", + " short_headline \\\n", + "0 1,5 Millionen Stimmberechtigte \n", + "1 Brasilien \n", + "2 Afghanistan-Politik der Bundesregierung \n", + "3 US-Haushaltsstreit \n", + "4 Strategiepapier \n", + "\n", + " short_text \\\n", + "0 Etwa 1,5 Millionen türkische Staatsbürger in D... \n", + "1 Brasiliens Ex-Präsident Bolsonaro hat eine Ver... \n", + "2 Deutschland hat seine Finanzhilfen für Afghani... \n", + "3 Die Republikaner im US-Kongress wollen Präside... \n", + "4 Russland will offenbar das Thema Ostseeverschm... \n", + "\n", + " article \\\n", + "0 Etwa 1,5 Millionen türkische Staatsbürger in D... \n", + "1 Brasiliens Ex-Präsident Bolsonaro hat eine Ver... \n", + "2 Deutschland hat seine Finanzhilfen für Afghani... \n", + "3 Die Republikaner im US-Kongress wollen Präside... \n", + "4 Russland will offenbar das Thema Ostseeverschm... \n", + "\n", + " link count \n", + "0 /ausland/europa/erdogan-wahlkampfpause-gesundh... 4 \n", + "1 /ausland/amerika/bolsonaro-brasilien-sturm-reg... 4 \n", + "2 /inland/innenpolitik/afghanistan-deutschland-h... 4 \n", + "3 /ausland/amerika/usa-haushaltsstreit-biden-rep... 4 \n", + "4 /investigativ/ndr-wdr/russland-kreml-strategie... 4 " + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "# filter out all rows where the count is not equal to 4\n", + "df = df[df[\"count\"] == 4]" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dateheadlineshort_headlineshort_textarticlelinkcount
02023-04-27Türkei-Wahl in Deutschland startet1,5 Millionen StimmberechtigteEtwa 1,5 Millionen türkische Staatsbürger in D...Etwa 1,5 Millionen türkische Staatsbürger in D.../ausland/europa/erdogan-wahlkampfpause-gesundh...4
12023-04-27Bolsonaro bestreitet Verwicklung in Regierungs...BrasilienBrasiliens Ex-Präsident Bolsonaro hat eine Ver...Brasiliens Ex-Präsident Bolsonaro hat eine Ver.../ausland/amerika/bolsonaro-brasilien-sturm-reg...4
22023-04-27Streiten, ob Hilfe wirklich richtig istAfghanistan-Politik der BundesregierungDeutschland hat seine Finanzhilfen für Afghani...Deutschland hat seine Finanzhilfen für Afghani.../inland/innenpolitik/afghanistan-deutschland-h...4
32023-04-27Republikaner machen Druck auf BidenUS-HaushaltsstreitDie Republikaner im US-Kongress wollen Präside...Die Republikaner im US-Kongress wollen Präside.../ausland/amerika/usa-haushaltsstreit-biden-rep...4
42023-04-27Russland plant Schein-Organisation zur Einflus...StrategiepapierRussland will offenbar das Thema Ostseeverschm...Russland will offenbar das Thema Ostseeverschm.../investigativ/ndr-wdr/russland-kreml-strategie...4
........................
217082018-02-01Die Zeit, die bleibtZeitumstellung wird überprüftEs wurde hitzig debattiert im EU-Parlament, da...Es wurde hitzig debattiert im EU-Parlament, da.../ausland/europa/zeitumstellung-129.html4
217182018-02-01SPD rutscht auf 18 Prozent abARD-DeutschlandTrendAuf ein Rekordtief von nur noch 18 Prozent fäl...Auf ein Rekordtief von nur noch 18 Prozent fäl.../inland/deutschlandtrend/deutschlandtrend-1109...4
217242018-02-0116 Prozent - SPD fällt auf RekordtiefDeutschlandTrend ExtraIm ARD-DeutschlandTrend kommt die SPD bei der ...Im ARD-DeutschlandTrend kommt die SPD bei der .../inland/deutschlandtrend/deutschlandtrend-1123...4
217892018-01-01SPD sackt auf 19 Prozent abARD-DeutschlandTrendSchlechte Nachrichten für die SPD: Im ARD-Deut...Schlechte Nachrichten für die SPD: Im ARD-Deut.../inland/deutschlandtrend/deutschlandtrend-1091...4
218062018-01-01Die Deutschen sind GroKo-müdeARD-DeutschlandTrendDie Mehrheit der Deutschen lehnt laut Deutschl...Die Mehrheit der Deutschen lehnt laut Deutschl.../inland/deutschlandtrend/deutschlandtrend-1077...4
\n", + "

12077 rows × 7 columns

\n", + "
" + ], + "text/plain": [ + " date headline \\\n", + "0 2023-04-27 Türkei-Wahl in Deutschland startet \n", + "1 2023-04-27 Bolsonaro bestreitet Verwicklung in Regierungs... \n", + "2 2023-04-27 Streiten, ob Hilfe wirklich richtig ist \n", + "3 2023-04-27 Republikaner machen Druck auf Biden \n", + "4 2023-04-27 Russland plant Schein-Organisation zur Einflus... \n", + "... ... ... \n", + "21708 2018-02-01 Die Zeit, die bleibt \n", + "21718 2018-02-01 SPD rutscht auf 18 Prozent ab \n", + "21724 2018-02-01 16 Prozent - SPD fällt auf Rekordtief \n", + "21789 2018-01-01 SPD sackt auf 19 Prozent ab \n", + "21806 2018-01-01 Die Deutschen sind GroKo-müde \n", + "\n", + " short_headline \\\n", + "0 1,5 Millionen Stimmberechtigte \n", + "1 Brasilien \n", + "2 Afghanistan-Politik der Bundesregierung \n", + "3 US-Haushaltsstreit \n", + "4 Strategiepapier \n", + "... ... \n", + "21708 Zeitumstellung wird überprüft \n", + "21718 ARD-DeutschlandTrend \n", + "21724 DeutschlandTrend Extra \n", + "21789 ARD-DeutschlandTrend \n", + "21806 ARD-DeutschlandTrend \n", + "\n", + " short_text \\\n", + "0 Etwa 1,5 Millionen türkische Staatsbürger in D... \n", + "1 Brasiliens Ex-Präsident Bolsonaro hat eine Ver... \n", + "2 Deutschland hat seine Finanzhilfen für Afghani... \n", + "3 Die Republikaner im US-Kongress wollen Präside... \n", + "4 Russland will offenbar das Thema Ostseeverschm... \n", + "... ... \n", + "21708 Es wurde hitzig debattiert im EU-Parlament, da... \n", + "21718 Auf ein Rekordtief von nur noch 18 Prozent fäl... \n", + "21724 Im ARD-DeutschlandTrend kommt die SPD bei der ... \n", + "21789 Schlechte Nachrichten für die SPD: Im ARD-Deut... \n", + "21806 Die Mehrheit der Deutschen lehnt laut Deutschl... \n", + "\n", + " article \\\n", + "0 Etwa 1,5 Millionen türkische Staatsbürger in D... \n", + "1 Brasiliens Ex-Präsident Bolsonaro hat eine Ver... \n", + "2 Deutschland hat seine Finanzhilfen für Afghani... \n", + "3 Die Republikaner im US-Kongress wollen Präside... \n", + "4 Russland will offenbar das Thema Ostseeverschm... \n", + "... ... \n", + "21708 Es wurde hitzig debattiert im EU-Parlament, da... \n", + "21718 Auf ein Rekordtief von nur noch 18 Prozent fäl... \n", + "21724 Im ARD-DeutschlandTrend kommt die SPD bei der ... \n", + "21789 Schlechte Nachrichten für die SPD: Im ARD-Deut... \n", + "21806 Die Mehrheit der Deutschen lehnt laut Deutschl... \n", + "\n", + " link count \n", + "0 /ausland/europa/erdogan-wahlkampfpause-gesundh... 4 \n", + "1 /ausland/amerika/bolsonaro-brasilien-sturm-reg... 4 \n", + "2 /inland/innenpolitik/afghanistan-deutschland-h... 4 \n", + "3 /ausland/amerika/usa-haushaltsstreit-biden-rep... 4 \n", + "4 /investigativ/ndr-wdr/russland-kreml-strategie... 4 \n", + "... ... ... \n", + "21708 /ausland/europa/zeitumstellung-129.html 4 \n", + "21718 /inland/deutschlandtrend/deutschlandtrend-1109... 4 \n", + "21724 /inland/deutschlandtrend/deutschlandtrend-1123... 4 \n", + "21789 /inland/deutschlandtrend/deutschlandtrend-1091... 4 \n", + "21806 /inland/deutschlandtrend/deutschlandtrend-1077... 4 \n", + "\n", + "[12077 rows x 7 columns]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "df[\"main_tag\"] = df[\"link\"].apply(lambda x: x.split(\"/\")[1])\n", + "df[\"sub_tag\"] = df[\"link\"].apply(lambda x: x.split(\"/\")[2])" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dateheadlineshort_headlineshort_textarticlelinkcountmain_tagsub_tag
02023-04-27Türkei-Wahl in Deutschland startet1,5 Millionen StimmberechtigteEtwa 1,5 Millionen türkische Staatsbürger in D...Etwa 1,5 Millionen türkische Staatsbürger in D.../ausland/europa/erdogan-wahlkampfpause-gesundh...4auslandeuropa
12023-04-27Bolsonaro bestreitet Verwicklung in Regierungs...BrasilienBrasiliens Ex-Präsident Bolsonaro hat eine Ver...Brasiliens Ex-Präsident Bolsonaro hat eine Ver.../ausland/amerika/bolsonaro-brasilien-sturm-reg...4auslandamerika
22023-04-27Streiten, ob Hilfe wirklich richtig istAfghanistan-Politik der BundesregierungDeutschland hat seine Finanzhilfen für Afghani...Deutschland hat seine Finanzhilfen für Afghani.../inland/innenpolitik/afghanistan-deutschland-h...4inlandinnenpolitik
32023-04-27Republikaner machen Druck auf BidenUS-HaushaltsstreitDie Republikaner im US-Kongress wollen Präside...Die Republikaner im US-Kongress wollen Präside.../ausland/amerika/usa-haushaltsstreit-biden-rep...4auslandamerika
42023-04-27Russland plant Schein-Organisation zur Einflus...StrategiepapierRussland will offenbar das Thema Ostseeverschm...Russland will offenbar das Thema Ostseeverschm.../investigativ/ndr-wdr/russland-kreml-strategie...4investigativndr-wdr
\n", + "
" + ], + "text/plain": [ + " date headline \\\n", + "0 2023-04-27 Türkei-Wahl in Deutschland startet \n", + "1 2023-04-27 Bolsonaro bestreitet Verwicklung in Regierungs... \n", + "2 2023-04-27 Streiten, ob Hilfe wirklich richtig ist \n", + "3 2023-04-27 Republikaner machen Druck auf Biden \n", + "4 2023-04-27 Russland plant Schein-Organisation zur Einflus... \n", + "\n", + " short_headline \\\n", + "0 1,5 Millionen Stimmberechtigte \n", + "1 Brasilien \n", + "2 Afghanistan-Politik der Bundesregierung \n", + "3 US-Haushaltsstreit \n", + "4 Strategiepapier \n", + "\n", + " short_text \\\n", + "0 Etwa 1,5 Millionen türkische Staatsbürger in D... \n", + "1 Brasiliens Ex-Präsident Bolsonaro hat eine Ver... \n", + "2 Deutschland hat seine Finanzhilfen für Afghani... \n", + "3 Die Republikaner im US-Kongress wollen Präside... \n", + "4 Russland will offenbar das Thema Ostseeverschm... \n", + "\n", + " article \\\n", + "0 Etwa 1,5 Millionen türkische Staatsbürger in D... \n", + "1 Brasiliens Ex-Präsident Bolsonaro hat eine Ver... \n", + "2 Deutschland hat seine Finanzhilfen für Afghani... \n", + "3 Die Republikaner im US-Kongress wollen Präside... \n", + "4 Russland will offenbar das Thema Ostseeverschm... \n", + "\n", + " link count main_tag \\\n", + "0 /ausland/europa/erdogan-wahlkampfpause-gesundh... 4 ausland \n", + "1 /ausland/amerika/bolsonaro-brasilien-sturm-reg... 4 ausland \n", + "2 /inland/innenpolitik/afghanistan-deutschland-h... 4 inland \n", + "3 /ausland/amerika/usa-haushaltsstreit-biden-rep... 4 ausland \n", + "4 /investigativ/ndr-wdr/russland-kreml-strategie... 4 investigativ \n", + "\n", + " sub_tag \n", + "0 europa \n", + "1 amerika \n", + "2 innenpolitik \n", + "3 amerika \n", + "4 ndr-wdr " + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "array(['ausland', 'inland', 'investigativ', 'wirtschaft', 'wissen',\n", + " 'multimedia', 'europawahl'], dtype=object)" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# print unique main tags\n", + "df[\"main_tag\"].unique()" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dateheadlineshort_headlineshort_textarticlelinkcountsub_tag
main_tag
ausland58025802580258025802580258025802
europawahl22222222
inland19731973197319731973197319731973
investigativ514514514514514514514514
multimedia4848484848484848
wirtschaft32023202320232023202320232023202
wissen536536536536536536536536
\n", + "
" + ], + "text/plain": [ + " date headline short_headline short_text article link \\\n", + "main_tag \n", + "ausland 5802 5802 5802 5802 5802 5802 \n", + "europawahl 2 2 2 2 2 2 \n", + "inland 1973 1973 1973 1973 1973 1973 \n", + "investigativ 514 514 514 514 514 514 \n", + "multimedia 48 48 48 48 48 48 \n", + "wirtschaft 3202 3202 3202 3202 3202 3202 \n", + "wissen 536 536 536 536 536 536 \n", + "\n", + " count sub_tag \n", + "main_tag \n", + "ausland 5802 5802 \n", + "europawahl 2 2 \n", + "inland 1973 1973 \n", + "investigativ 514 514 \n", + "multimedia 48 48 \n", + "wirtschaft 3202 3202 \n", + "wissen 536 536 " + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# group by main tag and count the number of rows\n", + "df.groupby(\"main_tag\").count()" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "main_tag\n", + "ausland [europa, amerika, afrika, asien, ozeanien, kor...\n", + "europawahl [hintergruende]\n", + "inland [innenpolitik, gesellschaft, deutschlandtrend,...\n", + "investigativ [ndr-wdr, kontraste, br-recherche, report-main...\n", + "multimedia [podcast]\n", + "wirtschaft [unternehmen, finanzen, konjunktur, verbrauche...\n", + "wissen [gesundheit, klima, forschung, technologie]\n", + "Name: sub_tag, dtype: object" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# group by main tag and print unique sub tags per main tag\n", + "df.groupby(\"main_tag\")[\"sub_tag\"].unique()" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dateheadlineshort_headlineshort_textarticlelinkcount
main_tagsub_tag
auslandafrika282282282282282282282
amerika1226122612261226122612261226
asien1370137013701370137013701370
europa2860286028602860286028602860
korrespondenten1111111
ozeanien62626262626262
uswahl1111111
europawahlhintergruende2222222
inlandbtw2136363636363636
deutschlandtrend102102102102102102102
gesellschaft550550550550550550550
innenpolitik1256125612561256125612561256
mittendrin27272727272727
regional1111111
wahlen1111111
investigativbr-recherche42424242424242
fakt8888888
funk7777777
hr1111111
hsb5555555
kontraste43434343434343
mdr6666666
monitor17171717171717
ndr75757575757575
ndr-wdr153153153153153153153
panorama18181818181818
rbb17171717171717
report-mainz29292929292929
report-muenchen8888888
swr37373737373737
wdr46464646464646
zapp2222222
multimediapodcast48484848484848
wirtschaftboerse39393939393939
finanzen279279279279279279279
konjunktur307307307307307307307
technologie248248248248248248248
unternehmen1160116011601160116011601160
verbraucher667667667667667667667
weltwirtschaft502502502502502502502
wissenforschung97979797979797
gesundheit144144144144144144144
klima241241241241241241241
technologie54545454545454
\n", + "
" + ], + "text/plain": [ + " date headline short_headline short_text \\\n", + "main_tag sub_tag \n", + "ausland afrika 282 282 282 282 \n", + " amerika 1226 1226 1226 1226 \n", + " asien 1370 1370 1370 1370 \n", + " europa 2860 2860 2860 2860 \n", + " korrespondenten 1 1 1 1 \n", + " ozeanien 62 62 62 62 \n", + " uswahl 1 1 1 1 \n", + "europawahl hintergruende 2 2 2 2 \n", + "inland btw21 36 36 36 36 \n", + " deutschlandtrend 102 102 102 102 \n", + " gesellschaft 550 550 550 550 \n", + " innenpolitik 1256 1256 1256 1256 \n", + " mittendrin 27 27 27 27 \n", + " regional 1 1 1 1 \n", + " wahlen 1 1 1 1 \n", + "investigativ br-recherche 42 42 42 42 \n", + " fakt 8 8 8 8 \n", + " funk 7 7 7 7 \n", + " hr 1 1 1 1 \n", + " hsb 5 5 5 5 \n", + " kontraste 43 43 43 43 \n", + " mdr 6 6 6 6 \n", + " monitor 17 17 17 17 \n", + " ndr 75 75 75 75 \n", + " ndr-wdr 153 153 153 153 \n", + " panorama 18 18 18 18 \n", + " rbb 17 17 17 17 \n", + " report-mainz 29 29 29 29 \n", + " report-muenchen 8 8 8 8 \n", + " swr 37 37 37 37 \n", + " wdr 46 46 46 46 \n", + " zapp 2 2 2 2 \n", + "multimedia podcast 48 48 48 48 \n", + "wirtschaft boerse 39 39 39 39 \n", + " finanzen 279 279 279 279 \n", + " konjunktur 307 307 307 307 \n", + " technologie 248 248 248 248 \n", + " unternehmen 1160 1160 1160 1160 \n", + " verbraucher 667 667 667 667 \n", + " weltwirtschaft 502 502 502 502 \n", + "wissen forschung 97 97 97 97 \n", + " gesundheit 144 144 144 144 \n", + " klima 241 241 241 241 \n", + " technologie 54 54 54 54 \n", + "\n", + " article link count \n", + "main_tag sub_tag \n", + "ausland afrika 282 282 282 \n", + " amerika 1226 1226 1226 \n", + " asien 1370 1370 1370 \n", + " europa 2860 2860 2860 \n", + " korrespondenten 1 1 1 \n", + " ozeanien 62 62 62 \n", + " uswahl 1 1 1 \n", + "europawahl hintergruende 2 2 2 \n", + "inland btw21 36 36 36 \n", + " deutschlandtrend 102 102 102 \n", + " gesellschaft 550 550 550 \n", + " innenpolitik 1256 1256 1256 \n", + " mittendrin 27 27 27 \n", + " regional 1 1 1 \n", + " wahlen 1 1 1 \n", + "investigativ br-recherche 42 42 42 \n", + " fakt 8 8 8 \n", + " funk 7 7 7 \n", + " hr 1 1 1 \n", + " hsb 5 5 5 \n", + " kontraste 43 43 43 \n", + " mdr 6 6 6 \n", + " monitor 17 17 17 \n", + " ndr 75 75 75 \n", + " ndr-wdr 153 153 153 \n", + " panorama 18 18 18 \n", + " rbb 17 17 17 \n", + " report-mainz 29 29 29 \n", + " report-muenchen 8 8 8 \n", + " swr 37 37 37 \n", + " wdr 46 46 46 \n", + " zapp 2 2 2 \n", + "multimedia podcast 48 48 48 \n", + "wirtschaft boerse 39 39 39 \n", + " finanzen 279 279 279 \n", + " konjunktur 307 307 307 \n", + " technologie 248 248 248 \n", + " unternehmen 1160 1160 1160 \n", + " verbraucher 667 667 667 \n", + " weltwirtschaft 502 502 502 \n", + "wissen forschung 97 97 97 \n", + " gesundheit 144 144 144 \n", + " klima 241 241 241 \n", + " technologie 54 54 54 " + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# group by main tag and sub tag and count the number of rows\n", + "df.groupby([\"main_tag\", \"sub_tag\"]).count()" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "# main tags to remove\n", + "main_tags_to_remove = [\"multimedia\", \"investigativ\", \"europawahl\"]\n", + "sub_tags_to_remove = [\"korrespondenten\", \"uswahl\", \"btw21\", \"regional\", \"wahlen\"]\n", + "\n", + "# filter out all rows where the main tag is in the list of main tags to remove\n", + "df = df[~df[\"main_tag\"].isin(main_tags_to_remove)]\n", + "\n", + "# filter out all rows where the sub tag is in the list of sub tags to remove\n", + "df = df[~df[\"sub_tag\"].isin(sub_tags_to_remove)]" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "main_tag sub_tag \n", + "ausland afrika 282\n", + " amerika 1226\n", + " asien 1370\n", + " europa 2860\n", + " ozeanien 62\n", + "inland deutschlandtrend 102\n", + " gesellschaft 550\n", + " innenpolitik 1256\n", + " mittendrin 27\n", + "wirtschaft boerse 39\n", + " finanzen 279\n", + " konjunktur 307\n", + " technologie 248\n", + " unternehmen 1160\n", + " verbraucher 667\n", + " weltwirtschaft 502\n", + "wissen forschung 97\n", + " gesundheit 144\n", + " klima 241\n", + " technologie 54\n", + "dtype: int64" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# group by main tag and sub tag and count the number of rows, but only show the count, not the other columns\n", + "df.groupby([\"main_tag\", \"sub_tag\"]).size()" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dateheadlineshort_headlineshort_textarticlelinkcountmain_tagsub_tag
02023-04-27Türkei-Wahl in Deutschland startet1,5 Millionen StimmberechtigteEtwa 1,5 Millionen türkische Staatsbürger in D...Etwa 1,5 Millionen türkische Staatsbürger in D.../ausland/europa/erdogan-wahlkampfpause-gesundh...4auslandeuropa
12023-04-27Bolsonaro bestreitet Verwicklung in Regierungs...BrasilienBrasiliens Ex-Präsident Bolsonaro hat eine Ver...Brasiliens Ex-Präsident Bolsonaro hat eine Ver.../ausland/amerika/bolsonaro-brasilien-sturm-reg...4auslandamerika
22023-04-27Streiten, ob Hilfe wirklich richtig istAfghanistan-Politik der BundesregierungDeutschland hat seine Finanzhilfen für Afghani...Deutschland hat seine Finanzhilfen für Afghani.../inland/innenpolitik/afghanistan-deutschland-h...4inlandinnenpolitik
32023-04-27Republikaner machen Druck auf BidenUS-HaushaltsstreitDie Republikaner im US-Kongress wollen Präside...Die Republikaner im US-Kongress wollen Präside.../ausland/amerika/usa-haushaltsstreit-biden-rep...4auslandamerika
72023-04-27Wie sich Twitter unter Elon Musk verändert hatEin halbes Jahr nach der Übernahme\"Der Vogel ist befreit\", twitterte Elon Musk v...\"Der Vogel ist befreit\", twitterte Elon Musk v.../wirtschaft/unternehmen/twitter-halbes-jahr-mu...4wirtschaftunternehmen
\n", + "
" + ], + "text/plain": [ + " date headline \\\n", + "0 2023-04-27 Türkei-Wahl in Deutschland startet \n", + "1 2023-04-27 Bolsonaro bestreitet Verwicklung in Regierungs... \n", + "2 2023-04-27 Streiten, ob Hilfe wirklich richtig ist \n", + "3 2023-04-27 Republikaner machen Druck auf Biden \n", + "7 2023-04-27 Wie sich Twitter unter Elon Musk verändert hat \n", + "\n", + " short_headline \\\n", + "0 1,5 Millionen Stimmberechtigte \n", + "1 Brasilien \n", + "2 Afghanistan-Politik der Bundesregierung \n", + "3 US-Haushaltsstreit \n", + "7 Ein halbes Jahr nach der Übernahme \n", + "\n", + " short_text \\\n", + "0 Etwa 1,5 Millionen türkische Staatsbürger in D... \n", + "1 Brasiliens Ex-Präsident Bolsonaro hat eine Ver... \n", + "2 Deutschland hat seine Finanzhilfen für Afghani... \n", + "3 Die Republikaner im US-Kongress wollen Präside... \n", + "7 \"Der Vogel ist befreit\", twitterte Elon Musk v... \n", + "\n", + " article \\\n", + "0 Etwa 1,5 Millionen türkische Staatsbürger in D... \n", + "1 Brasiliens Ex-Präsident Bolsonaro hat eine Ver... \n", + "2 Deutschland hat seine Finanzhilfen für Afghani... \n", + "3 Die Republikaner im US-Kongress wollen Präside... \n", + "7 \"Der Vogel ist befreit\", twitterte Elon Musk v... \n", + "\n", + " link count main_tag \\\n", + "0 /ausland/europa/erdogan-wahlkampfpause-gesundh... 4 ausland \n", + "1 /ausland/amerika/bolsonaro-brasilien-sturm-reg... 4 ausland \n", + "2 /inland/innenpolitik/afghanistan-deutschland-h... 4 inland \n", + "3 /ausland/amerika/usa-haushaltsstreit-biden-rep... 4 ausland \n", + "7 /wirtschaft/unternehmen/twitter-halbes-jahr-mu... 4 wirtschaft \n", + "\n", + " sub_tag \n", + "0 europa \n", + "1 amerika \n", + "2 innenpolitik \n", + "3 amerika \n", + "7 unternehmen " + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "# remove count column\n", + "df = df.drop(columns=[\"count\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "# combine main tag and sub tag\n", + "df[\"tag\"] = df[\"main_tag\"] + \"/\" + df[\"sub_tag\"]" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "# store the cleaned data in a new parquet file\n", + "df.to_parquet(\"tagesschau_cleaned.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
dateheadlineshort_headlineshort_textarticlelinkmain_tagsub_tagtag
02023-04-27Türkei-Wahl in Deutschland startet1,5 Millionen StimmberechtigteEtwa 1,5 Millionen türkische Staatsbürger in D...Etwa 1,5 Millionen türkische Staatsbürger in D.../ausland/europa/erdogan-wahlkampfpause-gesundh...auslandeuropaausland/europa
12023-04-27Bolsonaro bestreitet Verwicklung in Regierungs...BrasilienBrasiliens Ex-Präsident Bolsonaro hat eine Ver...Brasiliens Ex-Präsident Bolsonaro hat eine Ver.../ausland/amerika/bolsonaro-brasilien-sturm-reg...auslandamerikaausland/amerika
22023-04-27Streiten, ob Hilfe wirklich richtig istAfghanistan-Politik der BundesregierungDeutschland hat seine Finanzhilfen für Afghani...Deutschland hat seine Finanzhilfen für Afghani.../inland/innenpolitik/afghanistan-deutschland-h...inlandinnenpolitikinland/innenpolitik
32023-04-27Republikaner machen Druck auf BidenUS-HaushaltsstreitDie Republikaner im US-Kongress wollen Präside...Die Republikaner im US-Kongress wollen Präside.../ausland/amerika/usa-haushaltsstreit-biden-rep...auslandamerikaausland/amerika
72023-04-27Wie sich Twitter unter Elon Musk verändert hatEin halbes Jahr nach der Übernahme\"Der Vogel ist befreit\", twitterte Elon Musk v...\"Der Vogel ist befreit\", twitterte Elon Musk v.../wirtschaft/unternehmen/twitter-halbes-jahr-mu...wirtschaftunternehmenwirtschaft/unternehmen
\n", + "
" + ], + "text/plain": [ + " date headline \\\n", + "0 2023-04-27 Türkei-Wahl in Deutschland startet \n", + "1 2023-04-27 Bolsonaro bestreitet Verwicklung in Regierungs... \n", + "2 2023-04-27 Streiten, ob Hilfe wirklich richtig ist \n", + "3 2023-04-27 Republikaner machen Druck auf Biden \n", + "7 2023-04-27 Wie sich Twitter unter Elon Musk verändert hat \n", + "\n", + " short_headline \\\n", + "0 1,5 Millionen Stimmberechtigte \n", + "1 Brasilien \n", + "2 Afghanistan-Politik der Bundesregierung \n", + "3 US-Haushaltsstreit \n", + "7 Ein halbes Jahr nach der Übernahme \n", + "\n", + " short_text \\\n", + "0 Etwa 1,5 Millionen türkische Staatsbürger in D... \n", + "1 Brasiliens Ex-Präsident Bolsonaro hat eine Ver... \n", + "2 Deutschland hat seine Finanzhilfen für Afghani... \n", + "3 Die Republikaner im US-Kongress wollen Präside... \n", + "7 \"Der Vogel ist befreit\", twitterte Elon Musk v... \n", + "\n", + " article \\\n", + "0 Etwa 1,5 Millionen türkische Staatsbürger in D... \n", + "1 Brasiliens Ex-Präsident Bolsonaro hat eine Ver... \n", + "2 Deutschland hat seine Finanzhilfen für Afghani... \n", + "3 Die Republikaner im US-Kongress wollen Präside... \n", + "7 \"Der Vogel ist befreit\", twitterte Elon Musk v... \n", + "\n", + " link main_tag \\\n", + "0 /ausland/europa/erdogan-wahlkampfpause-gesundh... ausland \n", + "1 /ausland/amerika/bolsonaro-brasilien-sturm-reg... ausland \n", + "2 /inland/innenpolitik/afghanistan-deutschland-h... inland \n", + "3 /ausland/amerika/usa-haushaltsstreit-biden-rep... ausland \n", + "7 /wirtschaft/unternehmen/twitter-halbes-jahr-mu... wirtschaft \n", + "\n", + " sub_tag tag \n", + "0 europa ausland/europa \n", + "1 amerika ausland/amerika \n", + "2 innenpolitik inland/innenpolitik \n", + "3 amerika ausland/amerika \n", + "7 unternehmen wirtschaft/unternehmen " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.head()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/benchmarks/docker/.env.example b/benchmarks/docker/.env.example new file mode 100644 index 000000000..e729ee37d --- /dev/null +++ b/benchmarks/docker/.env.example @@ -0,0 +1,25 @@ +### Docker settings +COMPOSE_PROJECT_NAME=tim-dats-benchmarks + +# PostgreSQL +POSTGRES_USER=mlflow +POSTGRES_PASSWORD=mlflow +POSTGRES_DB=mlflow + +# S3 Credentials +AWS_ACCESS_KEY_ID=s3admin +AWS_SECRET_ACCESS_KEY=s3admin +AWS_DEFAULT_REGION=us-east-1 + +# RustFS +RUSTFS_CONSOLE_ENABLE=true +S3_BUCKET=mlflow + +# MLflow +MLFLOW_VERSION=latest +MLFLOW_HOST=0.0.0.0 +MLFLOW_PORT=5000 + +MLFLOW_BACKEND_STORE_URI=postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB} +MLFLOW_ARTIFACTS_DESTINATION=s3://${S3_BUCKET} +MLFLOW_S3_ENDPOINT_URL=http://storage:9000 diff --git a/benchmarks/docker/README.md b/benchmarks/docker/README.md new file mode 100644 index 000000000..a111fc576 --- /dev/null +++ b/benchmarks/docker/README.md @@ -0,0 +1,3 @@ +# MLFLOW + +See: https://github.com/mlflow/mlflow/tree/master/docker-compose diff --git a/benchmarks/docker/docker-compose.yml b/benchmarks/docker/docker-compose.yml new file mode 100644 index 000000000..d25c99b96 --- /dev/null +++ b/benchmarks/docker/docker-compose.yml @@ -0,0 +1,126 @@ +# FROM https://github.com/mlflow/mlflow/tree/master/docker-compose + +volumes: + pgdata: + storage-data: + +services: + postgres: + image: postgres:15 + container_name: mlflow-postgres + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + volumes: + - pgdata:/var/lib/postgresql/data + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 5s + timeout: 3s + retries: 10 + + storage: + image: rustfs/rustfs:1.0.0-alpha.83 + container_name: storage + environment: + RUSTFS_ADDRESS: :9000 + RUSTFS_SERVER_DOMAINS: storage:9000 + RUSTFS_REGION: ${AWS_DEFAULT_REGION:-us-east-1} + RUSTFS_ACCESS_KEY: ${AWS_ACCESS_KEY_ID:-s3admin} + RUSTFS_SECRET_KEY: ${AWS_SECRET_ACCESS_KEY:-s3admin} + RUSTFS_CONSOLE_ENABLE: ${RUSTFS_CONSOLE_ENABLE:-true} + ports: + - "9000:9000" + - "9001:9001" + volumes: + - storage-data:/data + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", 'curl -s http://127.0.0.1:9000/health | grep -q ''"status":"ok"'''] + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + + create-bucket: + image: amazon/aws-cli:2.33.25 + container_name: mlflow-create-bucket + depends_on: + storage: + condition: service_healthy + entrypoint: > + /bin/sh -c " + set -e; + echo 'Waiting for S3 gateway getting ready...'; + if aws --endpoint-url=${MLFLOW_S3_ENDPOINT_URL} s3api head-bucket --bucket ${S3_BUCKET} 2>/dev/null; then + echo 'Bucket ${S3_BUCKET} already exists. Skipping creation.'; + else + echo 'Creating bucket ${S3_BUCKET}...'; + aws --endpoint-url=${MLFLOW_S3_ENDPOINT_URL} s3api create-bucket --bucket ${S3_BUCKET} --region ${AWS_DEFAULT_REGION}; + fi + " + environment: + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION} + AWS_S3_ADDRESSING_STYLE: path + MLFLOW_S3_ENDPOINT_URL: ${MLFLOW_S3_ENDPOINT_URL} + S3_BUCKET: ${S3_BUCKET} + restart: "no" + + mlflow: + image: ghcr.io/mlflow/mlflow:${MLFLOW_VERSION} + container_name: mlflow-server + depends_on: + postgres: + condition: service_healthy + storage: + condition: service_healthy + create-bucket: + condition: service_completed_successfully + environment: + # Backend store URI built from vars + MLFLOW_BACKEND_STORE_URI: ${MLFLOW_BACKEND_STORE_URI} + + # S3/RustFS settings + MLFLOW_S3_ENDPOINT_URL: ${MLFLOW_S3_ENDPOINT_URL} + MLFLOW_ARTIFACTS_DESTINATION: ${MLFLOW_ARTIFACTS_DESTINATION} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION} + MLFLOW_S3_IGNORE_TLS: "true" + + # Server host/port + MLFLOW_HOST: ${MLFLOW_HOST} + MLFLOW_PORT: ${MLFLOW_PORT} + command: + - /bin/bash + - -c + - | + pip install --no-cache-dir psycopg2-binary boto3 + mlflow server \ + --backend-store-uri "${MLFLOW_BACKEND_STORE_URI}" \ + --artifacts-destination "${MLFLOW_ARTIFACTS_DESTINATION}" \ + --serve-artifacts \ + --host "${MLFLOW_HOST}" \ + --port "${MLFLOW_PORT}" + ports: + - "${MLFLOW_PORT}:${MLFLOW_PORT}" + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://localhost:${MLFLOW_PORT}/health')", + ] + interval: 10s + timeout: 5s + retries: 30 + +networks: + default: + name: mlflow-network diff --git a/benchmarks/pyproject.toml b/benchmarks/pyproject.toml new file mode 100644 index 000000000..809fa4976 --- /dev/null +++ b/benchmarks/pyproject.toml @@ -0,0 +1,45 @@ +[project] +name = "dats-llm-benchmarks" +version = "0.1.0" +description = "LLM benchmarking framework for NLP tasks in DATS" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "datasets>=3.1.0", + "docker>=7.1.0", + "evaluate>=0.4.2", + "hydra-core>=1.3.2", + "jinja2>=3.1.4", + "mlflow>=2.14.3", + "openai>=1.50.2", + "pandas>=2.2.3", + "pydantic>=2.9.2", + "pyright==1.1.385", + "pyyaml>=6.0.2", + "requests>=2.32.3", + "scikit-learn>=1.5.2", + "seqeval>=1.2.2", +] + +[tool.ruff] +extend = "../ruff.toml" +src = ["src"] + +[tool.ruff.lint.isort] +known-first-party = [ + "core", + "evaluation", + "prompts", + "schemas", +] + +[tool.pyright] +include = ["src", "test"] +exclude = [ + "**/__pycache__", + "**/.venv", +] +extraPaths = ["src"] +reportIncompatibleMethodOverride = false +reportIncompatibleVariableOverride = false +reportUnreachable = true diff --git a/benchmarks/src/.env.example b/benchmarks/src/.env.example new file mode 100644 index 000000000..884595127 --- /dev/null +++ b/benchmarks/src/.env.example @@ -0,0 +1 @@ +HF_TOKEN= diff --git a/benchmarks/src/core/docker_manager.py b/benchmarks/src/core/docker_manager.py new file mode 100644 index 000000000..c41e4fb23 --- /dev/null +++ b/benchmarks/src/core/docker_manager.py @@ -0,0 +1,119 @@ +import logging +import os +import time +from contextlib import contextmanager +from pathlib import Path +from typing import Iterator + +import docker +import requests +from docker.types import DeviceRequest + +from schemas.config.backend_schema import VllmBackendConfig +from schemas.config.model_schema import ModelConfig + +logger = logging.getLogger(__name__) + + +def _build_vllm_command(model_config: ModelConfig) -> str: + return ( + f"--model {model_config.name} " + f"--served-model-name {model_config.alias} " + f"--max-model-len {model_config.max_len} " + f"--gpu-memory-utilization {model_config.gpu_memory_utilization} " + f"--host 0.0.0.0" + ) + + +@contextmanager +def managed_vllm_container( + model_config: ModelConfig, + vllm_backend_config: VllmBackendConfig, + keep_on_failure: bool = True, # Useful for debugging +) -> Iterator[str]: + """Spin up a vLLM OpenAI-compatible API container and tear it down automatically. + + Yields: + Base URL of the running API server, e.g. "http://localhost:8000/v1". + """ + client = docker.from_env() + + hf_cache_host = Path(vllm_backend_config.hf_cache_dir).expanduser() + hf_cache_host.mkdir(parents=True, exist_ok=True) + + logger.info( + "Starting vLLM Docker container (image=%s, model=%s, host_port=%s, gpu_id=%s)", + vllm_backend_config.image, + model_config.name, + vllm_backend_config.host_port, + vllm_backend_config.gpu_id, + ) + + container = client.containers.run( + image=vllm_backend_config.image, + command=_build_vllm_command(model_config), + detach=True, + ports={"8000/tcp": vllm_backend_config.host_port}, + environment={ + "HUGGING_FACE_HUB_TOKEN": os.environ[vllm_backend_config.hf_token_env_var] + }, + device_requests=[ + DeviceRequest( + device_ids=[str(vllm_backend_config.gpu_id)], capabilities=[["gpu"]] + ) + ], + volumes={ + str(hf_cache_host): {"bind": "/root/.cache/huggingface", "mode": "rw"} + }, + ) + + health_url = f"http://localhost:{vllm_backend_config.host_port}/v1/models" + base_url = f"http://localhost:{vllm_backend_config.host_port}/v1" + + is_healthy = False + + try: + max_attempts = max(1, vllm_backend_config.startup_timeout_seconds // 2) + for _ in range(max_attempts): + # Check if container died prematurely to avoid waiting out the full timeout + container.reload() + if container.status == "exited": + logs = container.logs(tail=300).decode("utf-8", errors="ignore") + raise RuntimeError( + f"vLLM container crashed prematurely with exit code {container.attrs['State']['ExitCode']}.\n" + f"Recent container logs:\n{logs}" + ) + + try: + response = requests.get(health_url, timeout=2) + if response.status_code == 200: + is_healthy = True + logger.info("vLLM container became healthy at %s", base_url) + yield base_url + return + except requests.RequestException: + pass + + time.sleep(2) + + # If we exit the loop without returning, it timed out + logs = container.logs(tail=300).decode("utf-8", errors="ignore") + raise RuntimeError( + "vLLM container failed to become healthy within timeout. " + f"Health endpoint: {health_url}\n" + f"Recent container logs:\n{logs}" + ) + finally: + if not is_healthy and keep_on_failure: + print(f"Skipping cleanup for debugging. Container ID: {container.id}") + logger.warning( + "Keeping failed vLLM container alive for debugging: %s", container.id + ) + else: + logger.info("Stopping and removing vLLM container %s", container.id) + try: + container.stop(timeout=10) + except Exception: + pass + finally: + container.remove(force=True) diff --git a/benchmarks/src/core/llm_client.py b/benchmarks/src/core/llm_client.py new file mode 100644 index 000000000..b0e6af764 --- /dev/null +++ b/benchmarks/src/core/llm_client.py @@ -0,0 +1,73 @@ +import asyncio +from typing import Any + +from openai import AsyncOpenAI +from openai.types.chat import ChatCompletionMessageParam +from pydantic import BaseModel +from tqdm.asyncio import tqdm_asyncio + +from schemas.prediction.prediction_schema import BaseAnswerSchema + + +async def _fetch_completion( + client: AsyncOpenAI, + prompt: str, + system_prompt: str | None, + schema: type[BaseAnswerSchema], + model_alias: str, + temperature: float, +) -> str: + messages: list[ChatCompletionMessageParam] = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + messages.append({"role": "user", "content": prompt}) + + response = await client.chat.completions.create( + model=model_alias, + messages=messages, + temperature=temperature, + response_format={ + "type": "json_schema", + "json_schema": { + "name": schema.__name__, + "schema": schema.model_json_schema(), + "strict": True, + }, + }, + ) + + content: Any = response.choices[0].message.content + return "" if content is None else str(content).strip() + + +async def run_batch_inference( + prompts: list[str], + system_prompt: str | None, + schema: type[BaseAnswerSchema], + model_alias: str, + base_url: str, + api_key: str, + concurrency: int, + temperature: float, +) -> list[str]: + client = AsyncOpenAI(base_url=base_url, api_key=api_key) + semaphore = asyncio.Semaphore(concurrency) + + # Show the schema that is used for all following responses for better visibility in logs + print(f"Using response schema:\n{schema.model_json_schema()}\n{'-' * 60}") + + async def _run_one(prompt: str) -> str: + async with semaphore: + return await _fetch_completion( + client=client, + prompt=prompt, + system_prompt=system_prompt, + schema=schema, + model_alias=model_alias, + temperature=temperature, + ) + + tasks = [_run_one(prompt) for prompt in prompts] + return await tqdm_asyncio.gather( + *tasks, desc="Running inference", total=len(prompts) + ) diff --git a/benchmarks/src/core/logger.py b/benchmarks/src/core/logger.py new file mode 100644 index 000000000..7a5817420 --- /dev/null +++ b/benchmarks/src/core/logger.py @@ -0,0 +1,219 @@ +import json +import logging +import re +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import mlflow +import pandas as pd + +from schemas.config.run_schema import RunConfig + +logger = logging.getLogger(__name__) + + +def _slugify(value: str) -> str: + return re.sub(r"[^a-zA-Z0-9_-]+", "_", value).strip("_") + + +def _flatten_dict( + data: dict[str, Any], prefix: str = "" +) -> dict[str, str | float | int | bool]: + flattened: dict[str, str | float | int | bool] = {} + + for key, value in data.items(): + full_key = f"{prefix}.{key}" if prefix else str(key) + if isinstance(value, dict): + flattened.update(_flatten_dict(value, prefix=full_key)) + continue + + if isinstance(value, (str, int, float, bool)): + flattened[full_key] = value + continue + + flattened[full_key] = json.dumps(value, sort_keys=True) + + return flattened + + +def _write_local_artifacts( + output_dir: Path, + timestamp: str, + run_name: str, + metrics: dict[str, float], + config: dict[str, Any], + results_df: pd.DataFrame, +) -> tuple[Path, Path, Path]: + slug_run_name = _slugify(run_name) + csv_path = output_dir / f"{slug_run_name}_{timestamp}_details.csv" + metrics_path = output_dir / f"{slug_run_name}_{timestamp}_metrics.json" + config_path = output_dir / f"{slug_run_name}_{timestamp}_config.json" + + results_df.to_csv(csv_path, index=False) + metrics_path.write_text( + json.dumps(metrics, indent=2, sort_keys=True), encoding="utf-8" + ) + config_path.write_text( + json.dumps(config, indent=2, sort_keys=True), encoding="utf-8" + ) + + return csv_path, metrics_path, config_path + + +def _artifact_suffix_from_generated_name(artifact_path: Path) -> str: + name = artifact_path.name + if "_" not in name: + return name + return name.split("_", 1)[1] + + +def _rename_additional_artifacts_for_run( + artifact_paths: list[Path], + *, + run_name: str, + timestamp: str, +) -> list[Path]: + slug_run_name = _slugify(run_name) + renamed_paths: list[Path] = [] + + for artifact_path in artifact_paths: + if not artifact_path.exists(): + continue + + suffix = _artifact_suffix_from_generated_name(artifact_path) + target_path = artifact_path.with_name(f"{slug_run_name}_{timestamp}_{suffix}") + + # Avoid overwriting if the target already exists from another artifact. + collision_index = 1 + while target_path.exists() and target_path != artifact_path: + target_path = artifact_path.with_name( + f"{slug_run_name}_{timestamp}_{collision_index}_{suffix}" + ) + collision_index += 1 + + if target_path != artifact_path: + artifact_path = artifact_path.rename(target_path) + + renamed_paths.append(artifact_path) + + return renamed_paths + + +def log_experiment_to_mlflow( + run_config: RunConfig, + metrics: dict[str, float], + results_df: pd.DataFrame, + additional_artifact_paths: list[Path] | None = None, +) -> dict[str, Any]: + config = run_config.model_dump(mode="json") + output_dir: Path = run_config.output_dir + experiment_name = run_config.experiment.experiment_name + configured_run_name = run_config.experiment.run_name + mlflow_uri = run_config.mlflow_uri + + logger.info("Writing local benchmark artifacts to %s", output_dir) + + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + tracking_info: dict[str, Any] = { + "local_details_csv": None, + "local_metrics_json": None, + "local_config_json": None, + "local_additional_artifacts": [], + "effective_run_name": None, + "mlflow_logged": False, + "mlflow_run_id": None, + } + artifact_paths = additional_artifact_paths or [] + + try: + logger.info( + "Logging run '%s' to MLflow experiment '%s' at %s", + configured_run_name or "", + experiment_name, + mlflow_uri, + ) + mlflow.set_tracking_uri(str(mlflow_uri)) + mlflow.set_experiment(experiment_name) + + flat_params = _flatten_dict(config) + metric_values = {k: float(v) for k, v in metrics.items()} + + run_context = ( + mlflow.start_run(run_name=configured_run_name) + if configured_run_name + else mlflow.start_run() + ) + + with run_context as run: + # If run_name is omitted, MLflow auto-generates one. Reuse that name for local files. + effective_run_name = ( + run.data.tags.get("mlflow.runName") + or configured_run_name + or run.info.run_id + ) + csv_path, metrics_path, config_path = _write_local_artifacts( + output_dir=output_dir, + timestamp=timestamp, + run_name=effective_run_name, + metrics=metrics, + config=config, + results_df=results_df, + ) + + artifact_paths = _rename_additional_artifacts_for_run( + artifact_paths, + run_name=effective_run_name, + timestamp=timestamp, + ) + + mlflow.log_params(flat_params) + mlflow.log_metrics(metric_values) + mlflow.log_artifact(str(csv_path)) + mlflow.log_artifact(str(metrics_path)) + mlflow.log_artifact(str(config_path)) + for artifact_path in artifact_paths: + if artifact_path.exists(): + mlflow.log_artifact(str(artifact_path)) + + tracking_info["local_details_csv"] = str(csv_path) + tracking_info["local_metrics_json"] = str(metrics_path) + tracking_info["local_config_json"] = str(config_path) + tracking_info["local_additional_artifacts"] = [ + str(path) for path in artifact_paths if path.exists() + ] + tracking_info["effective_run_name"] = effective_run_name + tracking_info["mlflow_logged"] = True + tracking_info["mlflow_run_id"] = run.info.run_id + logger.info( + "MLflow run logged successfully: run_id=%s, run_name=%s", + run.info.run_id, + effective_run_name, + ) + except Exception as exc: + # Keep local artifacts even if MLflow fails. + effective_run_name = configured_run_name or "local_run" + csv_path, metrics_path, config_path = _write_local_artifacts( + output_dir=output_dir, + timestamp=timestamp, + run_name=effective_run_name, + metrics=metrics, + config=config, + results_df=results_df, + ) + artifact_paths = _rename_additional_artifacts_for_run( + artifact_paths, + run_name=effective_run_name, + timestamp=timestamp, + ) + tracking_info["local_details_csv"] = str(csv_path) + tracking_info["local_metrics_json"] = str(metrics_path) + tracking_info["local_config_json"] = str(config_path) + tracking_info["local_additional_artifacts"] = [ + str(path) for path in artifact_paths if path.exists() + ] + tracking_info["effective_run_name"] = effective_run_name + tracking_info["mlflow_error"] = str(exc) + logger.exception("MLflow logging failed: %s", exc) + + return tracking_info diff --git a/benchmarks/src/core/runner.py b/benchmarks/src/core/runner.py new file mode 100644 index 000000000..161b4358c --- /dev/null +++ b/benchmarks/src/core/runner.py @@ -0,0 +1,249 @@ +import asyncio +import logging +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import pandas as pd +from jinja2 import Environment, FileSystemLoader +from pydantic import ValidationError + +from core.docker_manager import managed_vllm_container +from core.llm_client import run_batch_inference +from core.logger import log_experiment_to_mlflow +from evaluation.artifact_registry import get_artifact_builders +from evaluation.metric_registry import get_metric_evaluators +from schemas.config.dataset_schema import DatasetConfig +from schemas.config.run_schema import RunConfig +from schemas.prediction.prediction_schema import BaseAnswerSchema +from schemas.prediction.schema_resolver import resolve_answer_schema + +logger = logging.getLogger(__name__) + + +def _load_answer_schema(answer_schema: str) -> type[BaseAnswerSchema]: + return resolve_answer_schema(answer_schema) + + +def _render_prompts( + df: pd.DataFrame, + template_dir: Path, + template_name: str, + dataset_config: DatasetConfig, + prompt_variables: dict[str, Any], +) -> list[str]: + env = Environment(loader=FileSystemLoader(str(template_dir)), autoescape=False) + template = env.get_template(template_name) + + prompts: list[str] = [] + for _, row in df.iterrows(): + row_data = {str(key): value for key, value in row.to_dict().items()} + row_context = dataset_config.build_prompt_row_context(row_data) + row_context.update(prompt_variables) + prompts.append(template.render(**row_context)) + + return prompts + + +def _render_system_prompt( + template_path: Path, + template_variables: dict[str, Any], +) -> str: + env = Environment( + loader=FileSystemLoader(str(template_path.parent)), + autoescape=False, + ) + template = env.get_template(template_path.name) + return template.render(**template_variables) + + +def _parse_responses( + raw_responses: list[str], + schema_class: type[BaseAnswerSchema], +) -> tuple[list[BaseAnswerSchema | None], list[bool], dict[int, str]]: + parsed_objects: list[BaseAnswerSchema | None] = [] + valid_flags: list[bool] = [] + parse_errors: dict[int, str] = {} + + for idx, response in enumerate(raw_responses): + try: + parsed_objects.append(schema_class.model_validate_json(response)) + valid_flags.append(True) + except ValidationError as exc: + parsed_objects.append(None) + valid_flags.append(False) + parse_errors[idx] = str(exc) + + return parsed_objects, valid_flags, parse_errors + + +def run_experiment(run_config: RunConfig) -> dict[str, Any]: + experiment_config = run_config.experiment + dataset_config = experiment_config.dataset + model_config = experiment_config.model + backend_config = run_config.backend + + logger.info( + "Loading dataset '%s' from %s", + dataset_config.name, + dataset_config.path, + ) + if dataset_config.path.suffix == ".csv": + df = pd.read_csv(dataset_config.path) + elif dataset_config.path.suffix == ".parquet": + df = pd.read_parquet(dataset_config.path) + else: + raise ValueError( + f"Unsupported dataset format: {dataset_config.path.suffix}. Supported formats are .csv and .parquet." + ) + + logger.info("Dataset loaded with %d rows", len(df)) + + if experiment_config.max_examples: + target_size = min(len(df), experiment_config.max_examples) + if experiment_config.sample_randomly: + df = df.sample( + n=target_size, + random_state=experiment_config.sample_random_state, + ).copy() + logger.info( + "Applying max_examples=%d with random sampling (seed=%d) -> %d rows", + experiment_config.max_examples, + experiment_config.sample_random_state, + len(df), + ) + else: + df = df.head(target_size).copy() + logger.info( + "Applying max_examples=%d -> %d rows", + experiment_config.max_examples, + len(df), + ) + + references = dataset_config.get_references(df) + + logger.info( + "Rendering prompts using template %s", experiment_config.prompt_template + ) + prompts = _render_prompts( + df=df, + template_dir=experiment_config.prompt_template.parent, + template_name=experiment_config.prompt_template.name, + dataset_config=dataset_config, + prompt_variables=experiment_config.prompt_variables, + ) + + print("Example rendered prompts:") + for i, prompt in enumerate(prompts[:3], start=1): + print(f"Prompt {i}:\n{prompt}\n{'-' * 60}") + + system_prompt: str | None = None + if experiment_config.system_prompt_template: + logger.info( + "Rendering system prompt using template %s", + experiment_config.system_prompt_template, + ) + system_prompt = _render_system_prompt( + template_path=experiment_config.system_prompt_template, + template_variables=experiment_config.system_prompt_variables, + ) + + print(f"Rendered system prompt:\n{system_prompt}\n{'-' * 60}") + + schema_class = _load_answer_schema(experiment_config.answer_schema) + logger.info("Loaded output schema %s", experiment_config.answer_schema) + + logger.info("Starting vLLM Docker container for model %s", model_config.name) + with managed_vllm_container( + model_config=model_config, + vllm_backend_config=backend_config, + ) as base_url: + logger.info("vLLM container is healthy at %s", base_url) + logger.info("Running batch inference for %d prompts", len(prompts)) + raw_responses = asyncio.run( + run_batch_inference( + prompts=prompts, + system_prompt=system_prompt, + schema=schema_class, + model_alias=model_config.alias, + base_url=base_url, + api_key=backend_config.api_key, + concurrency=backend_config.concurrency, + temperature=experiment_config.temperature, + ) + ) + + parsed_responses, valid_flags, parse_errors = _parse_responses( + raw_responses, schema_class + ) + + if parse_errors and run_config.fail_on_parse_error: + raise RuntimeError( + f"Schema validation failed for {len(parse_errors)} response(s)." + ) + + logger.info("Computing metrics: %s", experiment_config.metrics) + evaluators = get_metric_evaluators(metric_names=experiment_config.metrics) + + all_metrics: dict[str, float] = {} + for evaluator in evaluators: + all_metrics.update( + evaluator.compute(predictions=parsed_responses, references=references) + ) + all_metrics["parse_error_rate"] = ( + 0.0 + if not valid_flags + else 1.0 - (sum(1 for flag in valid_flags if flag) / len(valid_flags)) + ) + + results_data = dataset_config.log_dataset(df) + results_data.update( + { + "predicted_label": [ + parsed_response.get_prediction() + if parsed_response is not None + else None + for parsed_response in parsed_responses + ], + "prompt": prompts, + "raw_llm_response": raw_responses, + "parse_error": [parse_errors.get(i, "") for i in range(len(df))], + } + ) + results_df = pd.DataFrame(results_data) + + generated_artifact_paths: list[Path] = [] + if experiment_config.artifacts: + logger.info("Building artifacts: %s", experiment_config.artifacts) + artifact_builders = get_artifact_builders( + artifact_names=experiment_config.artifacts, + ) + # Use a temporary random prefix; final run-name-based filenames are assigned in logger. + artifact_prefix = uuid4().hex + for builder in artifact_builders: + generated_artifact_paths.extend( + builder.build( + predictions=parsed_responses, + references=references, + output_dir=run_config.output_dir, + artifact_prefix=artifact_prefix, + ) + ) + + logger.info("Logging artifacts and metrics to local outputs and MLflow") + tracking_info = log_experiment_to_mlflow( + run_config=run_config, + metrics=all_metrics, + results_df=results_df, + additional_artifact_paths=generated_artifact_paths, + ) + + return { + "num_examples": len(df), + "backend": "vllm", + "mlflow_experiment_name": experiment_config.experiment_name, + "mlflow_run_name": tracking_info.get("effective_run_name"), + "model_alias": model_config.alias, + "metrics": all_metrics, + "tracking": tracking_info, + } diff --git a/benchmarks/src/evaluation/artifact_base.py b/benchmarks/src/evaluation/artifact_base.py new file mode 100644 index 000000000..41b365585 --- /dev/null +++ b/benchmarks/src/evaluation/artifact_base.py @@ -0,0 +1,130 @@ +import logging +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Generic, Sequence, TypeVar, cast, get_args, get_origin + +from schemas.prediction.prediction_schema import BaseAnswerSchema +from schemas.reference.reference_schema import BaseReferenceSchema + +logger = logging.getLogger(__name__) + +AnswerSchemaT = TypeVar("AnswerSchemaT", bound=BaseAnswerSchema) +ReferenceSchemaT = TypeVar("ReferenceSchemaT", bound=BaseReferenceSchema) + + +class BaseArtifactBuilder(Generic[AnswerSchemaT, ReferenceSchemaT], ABC): + def __init__(self) -> None: + self.answer_schema_cls = self._required_answer_schema() + self.reference_schema_cls = self._required_reference_schema() + + def _required_answer_schema(self) -> type[AnswerSchemaT]: + for cls in type(self).mro(): + for base in getattr(cls, "__orig_bases__", ()): + origin = get_origin(base) + if isinstance(origin, type) and issubclass(origin, BaseArtifactBuilder): + args = get_args(base) + if ( + len(args) == 2 + and isinstance(args[0], type) + and issubclass(args[0], BaseAnswerSchema) + ): + return cast(type[AnswerSchemaT], args[0]) + + raise TypeError( + f"{self.__class__.__name__}: unable to resolve required schema type from generic BaseArtifactBuilder[...]." + ) + + def _required_reference_schema(self) -> type[ReferenceSchemaT]: + for cls in type(self).mro(): + for base in getattr(cls, "__orig_bases__", ()): + origin = get_origin(base) + if isinstance(origin, type) and issubclass(origin, BaseArtifactBuilder): + args = get_args(base) + if ( + len(args) == 2 + and isinstance(args[1], type) + and issubclass(args[1], BaseReferenceSchema) + ): + return cast(type[ReferenceSchemaT], args[1]) + + raise TypeError( + f"{self.__class__.__name__}: unable to resolve required reference schema type from generic BaseArtifactBuilder[...]." + ) + + def require_answer_schema( + self, + predictions: list[BaseAnswerSchema], + ) -> list[AnswerSchemaT]: + context = self.__class__.__name__ + typed_predictions: list[AnswerSchemaT] = [] + required_schema = self.answer_schema_cls + + for index, prediction in enumerate(predictions): + if not isinstance(prediction, required_schema): + raise TypeError( + f"{context}: expected {required_schema.__name__} predictions, " + f"got {type(prediction).__name__} at index {index}." + ) + + typed_predictions.append(cast(AnswerSchemaT, prediction)) + + return typed_predictions + + def require_reference_schema( + self, + references: Sequence[BaseReferenceSchema], + ) -> list[ReferenceSchemaT]: + typed_references: list[ReferenceSchemaT] = [] + required_schema = self.reference_schema_cls + + for reference in references: + if isinstance(reference, required_schema): + typed_references.append(cast(ReferenceSchemaT, reference)) + continue + + typed_references.append(required_schema.create_from_reference(reference)) + + return typed_references + + def discard_none_predictions( + self, + predictions: list[BaseAnswerSchema | None], + references: Sequence[BaseReferenceSchema], + ) -> tuple[list[BaseAnswerSchema], list[BaseReferenceSchema]]: + context = self.__class__.__name__ + if len(predictions) != len(references): + raise ValueError( + f"{context}: predictions and references must have identical length, " + f"got {len(predictions)} and {len(references)}." + ) + + filtered_predictions: list[BaseAnswerSchema] = [] + filtered_references: list[BaseReferenceSchema] = [] + discarded_count = 0 + + for prediction, reference in zip(predictions, references): + if prediction is None: + discarded_count += 1 + continue + + filtered_predictions.append(prediction) + filtered_references.append(reference) + + if discarded_count > 0: + logger.warning( + "%s: DISCARDED %d answers because prediction was None.", + context, + discarded_count, + ) + + return filtered_predictions, filtered_references + + @abstractmethod + def build( + self, + predictions: list[BaseAnswerSchema | None], + references: Sequence[BaseReferenceSchema], + output_dir: Path, + artifact_prefix: str, + ) -> list[Path]: + """Create artifacts and return local artifact file paths.""" diff --git a/benchmarks/src/evaluation/artifact_registry.py b/benchmarks/src/evaluation/artifact_registry.py new file mode 100644 index 000000000..7e4774d3a --- /dev/null +++ b/benchmarks/src/evaluation/artifact_registry.py @@ -0,0 +1,42 @@ +from typing import Any + +from evaluation.artifact_base import BaseArtifactBuilder +from evaluation.classification_artifacts import ( + MultiLabelClassificationReportArtifacts, + MultiLabelConfusionMatrixArtifacts, + SingleLabelClassificationReportArtifacts, + SingleLabelConfusionMatrixArtifacts, +) +from evaluation.sequential_sentence_classification_artifacts import ( + SequentialSentenceClassificationReportArtifacts, +) +from evaluation.span_classification_artifacts import SpanClassificationReportArtifacts + +ARTIFACT_REGISTRY: dict[str, type[BaseArtifactBuilder[Any, Any]]] = { + "classification_confusion_matrix": SingleLabelConfusionMatrixArtifacts, + "classification_report": SingleLabelClassificationReportArtifacts, + "multilabel_confusion_matrices": MultiLabelConfusionMatrixArtifacts, + "multilabel_classification_report": MultiLabelClassificationReportArtifacts, + "span_classification_report": SpanClassificationReportArtifacts, + "sequential_sentence_classification_report": SequentialSentenceClassificationReportArtifacts, +} + + +def get_artifact_builders( + artifact_names: list[str], +) -> list[BaseArtifactBuilder[Any, Any]]: + builders: list[BaseArtifactBuilder[Any, Any]] = [] + unknown_artifacts: list[str] = [] + + for name in artifact_names: + artifact_class = ARTIFACT_REGISTRY.get(name) + if artifact_class is None: + unknown_artifacts.append(name) + continue + builders.append(artifact_class()) + + if unknown_artifacts: + names = ", ".join(unknown_artifacts) + raise ValueError(f"Unknown artifact(s): {names}") + + return builders diff --git a/benchmarks/src/evaluation/classification_artifacts.py b/benchmarks/src/evaluation/classification_artifacts.py new file mode 100644 index 000000000..0bb92a34e --- /dev/null +++ b/benchmarks/src/evaluation/classification_artifacts.py @@ -0,0 +1,271 @@ +import re +from pathlib import Path +from typing import Sequence + +import matplotlib.pyplot as plt +import pandas as pd +from sklearn.metrics import ( + ConfusionMatrixDisplay, + classification_report, + confusion_matrix, + multilabel_confusion_matrix, +) +from sklearn.preprocessing import MultiLabelBinarizer + +from evaluation.artifact_base import BaseArtifactBuilder +from schemas.prediction.prediction_schema import ( + BaseAnswerSchema, + MultiLabelClassificationSchema, + SingleLabelClassificationSchema, +) +from schemas.reference.reference_schema import ( + BaseReferenceSchema, + MultiLabelReference, + SingleLabelReference, +) + + +def _slugify(value: str) -> str: + return re.sub(r"[^a-zA-Z0-9_-]+", "_", value).strip("_") + + +class SingleLabelConfusionMatrixArtifacts( + BaseArtifactBuilder[SingleLabelClassificationSchema, SingleLabelReference] +): + def build( + self, + predictions: list[BaseAnswerSchema | None], + references: Sequence[BaseReferenceSchema], + output_dir: Path, + artifact_prefix: str, + ) -> list[Path]: + filtered_predictions, filtered_references = self.discard_none_predictions( + predictions, + references, + ) + typed_predictions = self.require_answer_schema(filtered_predictions) + typed_references = self.require_reference_schema(filtered_references) + + pred_labels = [ + prediction.get_prediction().strip().lower() + for prediction in typed_predictions + ] + ref_labels = [reference.label.strip().lower() for reference in typed_references] + + if len(pred_labels) == 0: + return [] + + all_labels = sorted({*pred_labels, *ref_labels} - {""}) + if not all_labels: + return [] + + cm = confusion_matrix(ref_labels, pred_labels, labels=all_labels) + + csv_path = output_dir / f"{artifact_prefix}_confusion_matrix.csv" + png_path = output_dir / f"{artifact_prefix}_confusion_matrix.png" + + pd.DataFrame( + data=cm, + index=pd.Index(all_labels), + columns=pd.Index(all_labels), + ).to_csv(csv_path) + + fig, ax = plt.subplots(figsize=(max(6, len(all_labels) * 0.65), 6)) + disp = ConfusionMatrixDisplay(confusion_matrix=cm, display_labels=all_labels) + disp.plot(ax=ax, xticks_rotation="vertical", colorbar=False) + ax.set_title("Confusion Matrix") + fig.tight_layout() + fig.savefig(png_path) + plt.close(fig) + + return [csv_path, png_path] + + +class SingleLabelClassificationReportArtifacts( + BaseArtifactBuilder[SingleLabelClassificationSchema, SingleLabelReference] +): + def build( + self, + predictions: list[BaseAnswerSchema | None], + references: Sequence[BaseReferenceSchema], + output_dir: Path, + artifact_prefix: str, + ) -> list[Path]: + filtered_predictions, filtered_references = self.discard_none_predictions( + predictions, + references, + ) + typed_predictions = self.require_answer_schema(filtered_predictions) + typed_references = self.require_reference_schema(filtered_references) + + pred_labels = [ + prediction.get_prediction().strip().lower() + for prediction in typed_predictions + ] + ref_labels = [reference.label.strip().lower() for reference in typed_references] + + if len(pred_labels) == 0: + return [] + + all_labels = sorted({*pred_labels, *ref_labels} - {""}) + if not all_labels: + return [] + + report = classification_report( + ref_labels, + pred_labels, + labels=all_labels, + target_names=all_labels, + output_dict=True, + zero_division=0, + ) + + csv_path = output_dir / f"{artifact_prefix}_classification_report.csv" + pd.DataFrame(report).transpose().to_csv(csv_path) + return [csv_path] + + +class MultiLabelConfusionMatrixArtifacts( + BaseArtifactBuilder[MultiLabelClassificationSchema, MultiLabelReference] +): + def build( + self, + predictions: list[BaseAnswerSchema | None], + references: Sequence[BaseReferenceSchema], + output_dir: Path, + artifact_prefix: str, + ) -> list[Path]: + filtered_predictions, filtered_references = self.discard_none_predictions( + predictions, + references, + ) + typed_predictions = self.require_answer_schema(filtered_predictions) + typed_references = self.require_reference_schema(filtered_references) + + pred_label_lists = [ + [ + label.strip().lower() + for label in prediction.get_prediction() + if label.strip() + ] + for prediction in typed_predictions + ] + ref_label_lists = [ + [label.strip().lower() for label in typed_reference.labels if label.strip()] + for typed_reference in typed_references + ] + + if len(pred_label_lists) == 0: + return [] + + label_names = sorted( + { + label + for labels in (pred_label_lists + ref_label_lists) + for label in labels + if label + } + ) + if not label_names: + return [] + + mlb = MultiLabelBinarizer(classes=label_names) + mlb.fit([label_names]) + + y_true = mlb.transform(ref_label_lists) + y_pred = mlb.transform(pred_label_lists) + cms = multilabel_confusion_matrix( + y_true, y_pred, labels=range(len(label_names)) + ) + + artifact_paths: list[Path] = [] + for idx, label_name in enumerate(label_names): + label_slug = _slugify(label_name) + csv_path = ( + output_dir / f"{artifact_prefix}_confusion_matrix_{label_slug}.csv" + ) + png_path = ( + output_dir / f"{artifact_prefix}_confusion_matrix_{label_slug}.png" + ) + + cm = cms[idx] + pd.DataFrame( + data=cm, + index=pd.Index(["actual_negative", "actual_positive"]), + columns=pd.Index(["pred_negative", "pred_positive"]), + ).to_csv(csv_path) + + fig, ax = plt.subplots(figsize=(5, 5)) + disp = ConfusionMatrixDisplay(confusion_matrix=cm) + disp.plot(ax=ax, colorbar=False) + ax.set_title(f"Confusion Matrix: {label_name}") + fig.tight_layout() + fig.savefig(png_path) + plt.close(fig) + + artifact_paths.extend([csv_path, png_path]) + + return artifact_paths + + +class MultiLabelClassificationReportArtifacts( + BaseArtifactBuilder[MultiLabelClassificationSchema, MultiLabelReference] +): + def build( + self, + predictions: list[BaseAnswerSchema | None], + references: Sequence[BaseReferenceSchema], + output_dir: Path, + artifact_prefix: str, + ) -> list[Path]: + filtered_predictions, filtered_references = self.discard_none_predictions( + predictions, + references, + ) + typed_predictions = self.require_answer_schema(filtered_predictions) + typed_references = self.require_reference_schema(filtered_references) + + pred_label_lists = [ + [ + label.strip().lower() + for label in prediction.get_prediction() + if label.strip() + ] + for prediction in typed_predictions + ] + ref_label_lists = [ + [label.strip().lower() for label in typed_reference.labels if label.strip()] + for typed_reference in typed_references + ] + + if len(pred_label_lists) == 0: + return [] + + label_names = sorted( + { + label + for labels in (pred_label_lists + ref_label_lists) + for label in labels + if label + } + ) + if not label_names: + return [] + + mlb = MultiLabelBinarizer(classes=label_names) + mlb.fit([label_names]) + + y_true = mlb.transform(ref_label_lists) + y_pred = mlb.transform(pred_label_lists) + + report = classification_report( + y_true, + y_pred, + target_names=label_names, + output_dict=True, + zero_division=0, + ) + + csv_path = output_dir / f"{artifact_prefix}_classification_report.csv" + pd.DataFrame(report).transpose().to_csv(csv_path) + return [csv_path] diff --git a/benchmarks/src/evaluation/classification_metrics.py b/benchmarks/src/evaluation/classification_metrics.py new file mode 100644 index 000000000..cf572ced2 --- /dev/null +++ b/benchmarks/src/evaluation/classification_metrics.py @@ -0,0 +1,179 @@ +from typing import Sequence + +from sklearn.metrics import accuracy_score, precision_recall_fscore_support +from sklearn.preprocessing import MultiLabelBinarizer + +from evaluation.metric_base import BaseMetricWrapper +from schemas.prediction.prediction_schema import ( + BaseAnswerSchema, + MultiLabelClassificationSchema, + SingleLabelClassificationSchema, +) +from schemas.reference.reference_schema import ( + BaseReferenceSchema, + MultiLabelReference, + SingleLabelReference, +) + + +class StandardClassificationMetrics( + BaseMetricWrapper[SingleLabelClassificationSchema, SingleLabelReference] +): + def compute( + self, + predictions: list[BaseAnswerSchema | None], + references: Sequence[BaseReferenceSchema], + ) -> dict[str, float]: + filtered_predictions, filtered_references = self.discard_none_predictions( + predictions, + references, + ) + typed_predictions = self.require_answer_schema(filtered_predictions) + typed_references = self.require_reference_schema(filtered_references) + + pred_labels = [ + prediction.get_prediction().strip().lower() + for prediction in typed_predictions + ] + ref_labels = [reference.label.strip().lower() for reference in typed_references] + + if len(pred_labels) == 0: + return { + "accuracy": 0.0, + "macro_precision": 0.0, + "macro_recall": 0.0, + "macro_f1": 0.0, + } + + accuracy = accuracy_score(ref_labels, pred_labels) + precision, recall, f1, _ = precision_recall_fscore_support( + ref_labels, + pred_labels, + average="macro", + zero_division=0, + ) + + return { + "accuracy": float(accuracy), + "macro_precision": float(precision), + "macro_recall": float(recall), + "macro_f1": float(f1), + } + + +class WeightedClassificationMetrics( + BaseMetricWrapper[SingleLabelClassificationSchema, SingleLabelReference] +): + def compute( + self, + predictions: list[BaseAnswerSchema | None], + references: Sequence[BaseReferenceSchema], + ) -> dict[str, float]: + filtered_predictions, filtered_references = self.discard_none_predictions( + predictions, + references, + ) + typed_predictions = self.require_answer_schema(filtered_predictions) + typed_references = self.require_reference_schema(filtered_references) + + pred_labels = [ + prediction.get_prediction().strip().lower() + for prediction in typed_predictions + ] + ref_labels = [reference.label.strip().lower() for reference in typed_references] + + if len(pred_labels) == 0: + return { + "weighted_precision": 0.0, + "weighted_recall": 0.0, + "weighted_f1": 0.0, + "weighted_accuracy": 0.0, + } + + accuracy = accuracy_score(ref_labels, pred_labels) + precision, recall, f1, _ = precision_recall_fscore_support( + ref_labels, + pred_labels, + average="weighted", + zero_division=0, + ) + + return { + "weighted_precision": float(precision), + "weighted_recall": float(recall), + "weighted_f1": float(f1), + "weighted_accuracy": float(accuracy), + } + + +class MultiLabelClassificationMetrics( + BaseMetricWrapper[MultiLabelClassificationSchema, MultiLabelReference] +): + def compute( + self, + predictions: list[BaseAnswerSchema | None], + references: Sequence[BaseReferenceSchema], + ) -> dict[str, float]: + filtered_predictions, filtered_references = self.discard_none_predictions( + predictions, + references, + ) + typed_predictions = self.require_answer_schema(filtered_predictions) + typed_references = self.require_reference_schema(filtered_references) + + pred_labels = [ + [ + label.strip().lower() + for label in prediction.get_prediction() + if label.strip() + ] + for prediction in typed_predictions + ] + ref_labels = [ + [label.strip().lower() for label in reference.labels if label.strip()] + for reference in typed_references + ] + + if len(pred_labels) == 0: + return { + "weighted_precision": 0.0, + "weighted_recall": 0.0, + "weighted_f1": 0.0, + "subset_accuracy": 0.0, + } + + label_names = sorted( + { + label + for labels in (pred_labels + ref_labels) + for label in labels + if label + } + ) + if not label_names: + return { + "weighted_precision": 0.0, + "weighted_recall": 0.0, + "weighted_f1": 0.0, + "subset_accuracy": 0.0, + } + + mlb = MultiLabelBinarizer(classes=label_names) + mlb.fit([label_names]) + y_true = mlb.transform(ref_labels) + y_pred = mlb.transform(pred_labels) + + precision, recall, f1, _ = precision_recall_fscore_support( + y_true, + y_pred, + average="weighted", + zero_division=0, + ) + subset_accuracy = accuracy_score(y_true, y_pred) + + return { + "weighted_precision": float(precision), + "weighted_recall": float(recall), + "weighted_f1": float(f1), + "subset_accuracy": float(subset_accuracy), + } diff --git a/benchmarks/src/evaluation/eval_utils.py b/benchmarks/src/evaluation/eval_utils.py new file mode 100644 index 000000000..2419c2c43 --- /dev/null +++ b/benchmarks/src/evaluation/eval_utils.py @@ -0,0 +1,54 @@ +from typing import Any + + +def _normalize_label(value: str) -> str: + return value.strip().lower() + + +def _to_label(value: Any, normalize: bool = False) -> str: + if value is None: + return "" + + if isinstance(value, (list, tuple, set)): + label = ", ".join(str(item) for item in value) + else: + label = str(value) + + return _normalize_label(label) if normalize else label + + +def _to_label_list(value: Any, normalize: bool = False) -> list[str]: + if value is None: + return [] + + if isinstance(value, str): + labels = [part.strip() for part in value.split(",") if part.strip()] + elif isinstance(value, (list, tuple, set)): + labels = [str(item).strip() for item in value if str(item).strip()] + else: + labels = [str(value).strip()] if str(value).strip() else [] + + if normalize: + return [_normalize_label(label) for label in labels] + + return labels + + +def extract_labels(values: list[Any], normalize: bool = False) -> list[str]: + return [ + _to_label( + item, + normalize=normalize, + ) + for item in values + ] + + +def extract_multilabels(values: list[Any], normalize: bool = False) -> list[list[str]]: + return [ + _to_label_list( + item, + normalize=normalize, + ) + for item in values + ] diff --git a/benchmarks/src/evaluation/extractive_qa_metrics.py b/benchmarks/src/evaluation/extractive_qa_metrics.py new file mode 100644 index 000000000..0fbb0d322 --- /dev/null +++ b/benchmarks/src/evaluation/extractive_qa_metrics.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from typing import Any, Sequence + +import evaluate + +from evaluation.metric_base import BaseMetricWrapper +from schemas.prediction.prediction_schema import ( + BaseAnswerSchema, + ExtractiveQASchema, +) +from schemas.reference.reference_schema import ( + BaseReferenceSchema, + ExtractiveQAReference, +) + +_NO_ANSWER_MARKERS = ("not answerable", "nicht beantwortbar") + + +def _is_no_answer(answer: str) -> bool: + normalized = answer.strip().lower() + if not normalized: + return False + return normalized in _NO_ANSWER_MARKERS + + +class ExtractiveQASquad2Metrics( + BaseMetricWrapper[ExtractiveQASchema, ExtractiveQAReference] +): + def __init__(self) -> None: + super().__init__() + self.metric = evaluate.load("squad_v2") + + def compute( + self, + predictions: list[BaseAnswerSchema | None], + references: Sequence[BaseReferenceSchema], + ) -> dict[str, float]: + filtered_predictions, filtered_references = self.discard_none_predictions( + predictions, + references, + ) + typed_predictions = self.require_answer_schema(filtered_predictions) + typed_references = self.require_reference_schema(filtered_references) + + if len(typed_predictions) == 0: + raise ValueError("predictions and references must not be empty.") + + evaluation_predictions: list[dict[str, Any]] = [] + evaluation_references: list[dict[str, Any]] = [] + + for parsed_object, reference in zip(typed_predictions, typed_references): + answer = parsed_object.get_prediction() + + no_answer_probability = 1.0 if _is_no_answer(answer) else 0.0 + + evaluation_predictions.append( + { + "id": reference.id, + "prediction_text": "" if no_answer_probability == 1.0 else answer, + "no_answer_probability": no_answer_probability, + } + ) + evaluation_references.append(reference.model_dump()) + + results = self.metric.compute( + predictions=evaluation_predictions, + references=evaluation_references, + ) + if results is None: + raise ValueError("squad_v2 metric returned None.") + + exact = float(results.get("exact", 0.0)) + f1 = float(results.get("f1", 0.0)) + + return { + "exact_match": round(exact, 2), + "f1": round(f1, 2), + } diff --git a/benchmarks/src/evaluation/metric_base.py b/benchmarks/src/evaluation/metric_base.py new file mode 100644 index 000000000..18bb1bb5b --- /dev/null +++ b/benchmarks/src/evaluation/metric_base.py @@ -0,0 +1,127 @@ +import logging +from abc import ABC, abstractmethod +from typing import Generic, Sequence, TypeVar, cast, get_args, get_origin + +from schemas.prediction.prediction_schema import BaseAnswerSchema +from schemas.reference.reference_schema import BaseReferenceSchema + +logger = logging.getLogger(__name__) + +AnswerSchemaT = TypeVar("AnswerSchemaT", bound=BaseAnswerSchema) +ReferenceSchemaT = TypeVar("ReferenceSchemaT", bound=BaseReferenceSchema) + + +class BaseMetricWrapper(Generic[AnswerSchemaT, ReferenceSchemaT], ABC): + def __init__(self) -> None: + self.answer_schema_cls = self._required_answer_schema() + self.reference_schema_cls = self._required_reference_schema() + + def _required_answer_schema(self) -> type[AnswerSchemaT]: + for cls in type(self).mro(): + for base in getattr(cls, "__orig_bases__", ()): + origin = get_origin(base) + if isinstance(origin, type) and issubclass(origin, BaseMetricWrapper): + args = get_args(base) + if ( + len(args) == 2 + and isinstance(args[0], type) + and issubclass(args[0], BaseAnswerSchema) + ): + return cast(type[AnswerSchemaT], args[0]) + + raise TypeError( + f"{self.__class__.__name__}: unable to resolve required schema type from generic BaseMetricWrapper[...]." + ) + + def _required_reference_schema(self) -> type[ReferenceSchemaT]: + for cls in type(self).mro(): + for base in getattr(cls, "__orig_bases__", ()): + origin = get_origin(base) + if isinstance(origin, type) and issubclass(origin, BaseMetricWrapper): + args = get_args(base) + if ( + len(args) == 2 + and isinstance(args[1], type) + and issubclass(args[1], BaseReferenceSchema) + ): + return cast(type[ReferenceSchemaT], args[1]) + + raise TypeError( + f"{self.__class__.__name__}: unable to resolve required reference schema type from generic BaseMetricWrapper[...]." + ) + + def require_answer_schema( + self, + predictions: list[BaseAnswerSchema], + ) -> list[AnswerSchemaT]: + context = self.__class__.__name__ + typed_predictions: list[AnswerSchemaT] = [] + required_schema = self.answer_schema_cls + + for index, prediction in enumerate(predictions): + if not isinstance(prediction, required_schema): + raise TypeError( + f"{context}: expected {required_schema.__name__} predictions, " + f"got {type(prediction).__name__} at index {index}." + ) + + typed_predictions.append(cast(AnswerSchemaT, prediction)) + + return typed_predictions + + def require_reference_schema( + self, + references: Sequence[BaseReferenceSchema], + ) -> list[ReferenceSchemaT]: + typed_references: list[ReferenceSchemaT] = [] + required_schema = self.reference_schema_cls + + for reference in references: + if isinstance(reference, required_schema): + typed_references.append(cast(ReferenceSchemaT, reference)) + continue + + typed_references.append(required_schema.create_from_reference(reference)) + + return typed_references + + def discard_none_predictions( + self, + predictions: list[BaseAnswerSchema | None], + references: Sequence[BaseReferenceSchema], + ) -> tuple[list[BaseAnswerSchema], list[BaseReferenceSchema]]: + context = self.__class__.__name__ + if len(predictions) != len(references): + raise ValueError( + f"{context}: predictions and references must have identical length, " + f"got {len(predictions)} and {len(references)}." + ) + + filtered_predictions: list[BaseAnswerSchema] = [] + filtered_references: list[BaseReferenceSchema] = [] + discarded_count = 0 + + for prediction, reference in zip(predictions, references): + if prediction is None: + discarded_count += 1 + continue + + filtered_predictions.append(prediction) + filtered_references.append(reference) + + if discarded_count > 0: + logger.warning( + "%s: DISCARDED %d answers because prediction was None.", + context, + discarded_count, + ) + + return filtered_predictions, filtered_references + + @abstractmethod + def compute( + self, + predictions: list[BaseAnswerSchema | None], + references: Sequence[BaseReferenceSchema], + ) -> dict[str, float]: + """Return metric names and values for a prediction/reference pair list.""" diff --git a/benchmarks/src/evaluation/metric_registry.py b/benchmarks/src/evaluation/metric_registry.py new file mode 100644 index 000000000..03abfc6aa --- /dev/null +++ b/benchmarks/src/evaluation/metric_registry.py @@ -0,0 +1,44 @@ +from typing import Any + +from evaluation.classification_metrics import ( + MultiLabelClassificationMetrics, + StandardClassificationMetrics, + WeightedClassificationMetrics, +) +from evaluation.extractive_qa_metrics import ExtractiveQASquad2Metrics +from evaluation.metric_base import BaseMetricWrapper +from evaluation.sequential_sentence_classification_metrics import ( + SequentialSentenceClassificationMetrics, +) +from evaluation.span_classification_metrics import SpanClassificationMetrics +from evaluation.template_filling_metrics import TemplateFillingMUC4Metrics + +METRIC_REGISTRY: dict[str, type[BaseMetricWrapper[Any, Any]]] = { + "classification_macro_metrics": StandardClassificationMetrics, + "classification_weighted_metrics": WeightedClassificationMetrics, + "multilabel_weighted_metrics": MultiLabelClassificationMetrics, + "extractive_qa_squad2_metrics": ExtractiveQASquad2Metrics, + "template_filling_muc4_metrics": TemplateFillingMUC4Metrics, + "span_classification_metrics": SpanClassificationMetrics, + "sequential_sentence_classification_metrics": SequentialSentenceClassificationMetrics, +} + + +def get_metric_evaluators( + metric_names: list[str], +) -> list[BaseMetricWrapper[Any, Any]]: + evaluators: list[BaseMetricWrapper[Any, Any]] = [] + unknown_metrics: list[str] = [] + + for name in metric_names: + metric_class = METRIC_REGISTRY.get(name) + if metric_class is None: + unknown_metrics.append(name) + continue + evaluators.append(metric_class()) + + if unknown_metrics: + names = ", ".join(unknown_metrics) + raise ValueError(f"Unknown metric(s): {names}") + + return evaluators diff --git a/benchmarks/src/evaluation/sequential_sentence_classification_artifacts.py b/benchmarks/src/evaluation/sequential_sentence_classification_artifacts.py new file mode 100644 index 000000000..139ffba6c --- /dev/null +++ b/benchmarks/src/evaluation/sequential_sentence_classification_artifacts.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Sequence + +import pandas as pd +from seqeval.metrics import classification_report + +from evaluation.artifact_base import BaseArtifactBuilder +from evaluation.sequential_sentence_classification_utils import ( + build_label_sequences, + to_bio_format, +) +from schemas.prediction.prediction_schema import ( + BaseAnswerSchema, + SequentialSentenceClassificationSchema, +) +from schemas.reference.reference_schema import ( + BaseReferenceSchema, + SequentialSentenceClassificationReference, +) + + +class SequentialSentenceClassificationReportArtifacts( + BaseArtifactBuilder[ + SequentialSentenceClassificationSchema, + SequentialSentenceClassificationReference, + ] +): + def build( + self, + predictions: list[BaseAnswerSchema | None], + references: Sequence[BaseReferenceSchema], + output_dir: Path, + artifact_prefix: str, + ) -> list[Path]: + filtered_predictions, filtered_references = self.discard_none_predictions( + predictions, + references, + ) + typed_predictions = self.require_answer_schema(filtered_predictions) + typed_references = self.require_reference_schema(filtered_references) + + if len(typed_predictions) == 0: + return [] + + gold_sequences, pred_sequences = build_label_sequences( + predictions=typed_predictions, + references=typed_references, + ) + if sum(len(sequence) for sequence in gold_sequences) == 0: + return [] + + gold_sequences_bio = to_bio_format(gold_sequences) + pred_sequences_bio = to_bio_format(pred_sequences) + + report = classification_report( + gold_sequences_bio, + pred_sequences_bio, + output_dict=True, + ) + csv_path = ( + output_dir + / f"{artifact_prefix}_sequential_sentence_classification_report.csv" + ) + pd.DataFrame(report).transpose().to_csv(csv_path) + + return [csv_path] diff --git a/benchmarks/src/evaluation/sequential_sentence_classification_metrics.py b/benchmarks/src/evaluation/sequential_sentence_classification_metrics.py new file mode 100644 index 000000000..2efc333c5 --- /dev/null +++ b/benchmarks/src/evaluation/sequential_sentence_classification_metrics.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from typing import Any, Sequence, cast + +from seqeval.metrics import accuracy_score, classification_report, f1_score + +from evaluation.metric_base import BaseMetricWrapper +from evaluation.sequential_sentence_classification_utils import ( + build_label_sequences, + to_bio_format, +) +from schemas.prediction.prediction_schema import ( + BaseAnswerSchema, + SequentialSentenceClassificationSchema, +) +from schemas.reference.reference_schema import ( + BaseReferenceSchema, + SequentialSentenceClassificationReference, +) + + +class SequentialSentenceClassificationMetrics( + BaseMetricWrapper[ + SequentialSentenceClassificationSchema, + SequentialSentenceClassificationReference, + ] +): + def compute( + self, + predictions: list[BaseAnswerSchema | None], + references: Sequence[BaseReferenceSchema], + ) -> dict[str, float]: + filtered_predictions, filtered_references = self.discard_none_predictions( + predictions, + references, + ) + typed_predictions = self.require_answer_schema(filtered_predictions) + typed_references = self.require_reference_schema(filtered_references) + + if len(typed_predictions) == 0: + return { + "precision": 0.0, + "recall": 0.0, + "f1": 0.0, + "accuracy": 0.0, + } + + gold_sequences, pred_sequences = build_label_sequences( + predictions=typed_predictions, + references=typed_references, + ) + if sum(len(sequence) for sequence in gold_sequences) == 0: + return { + "precision": 0.0, + "recall": 0.0, + "f1": 0.0, + "accuracy": 0.0, + } + + gold_sequences_bio = to_bio_format(gold_sequences) + pred_sequences_bio = to_bio_format(pred_sequences) + + accuracy = float(accuracy_score(gold_sequences_bio, pred_sequences_bio)) + f1 = float( + cast( + float, + f1_score(gold_sequences_bio, pred_sequences_bio), + ) + ) + + report = classification_report( + gold_sequences_bio, + pred_sequences_bio, + output_dict=True, + ) + report_dict = cast(dict[str, Any], report) + weighted_report = cast(dict[str, Any], report_dict.get("weighted avg", {})) + + return { + "precision": float(weighted_report.get("precision", 0.0)), + "recall": float(weighted_report.get("recall", 0.0)), + "f1": f1, + "accuracy": accuracy, + } diff --git a/benchmarks/src/evaluation/sequential_sentence_classification_utils.py b/benchmarks/src/evaluation/sequential_sentence_classification_utils.py new file mode 100644 index 000000000..07b92e9d4 --- /dev/null +++ b/benchmarks/src/evaluation/sequential_sentence_classification_utils.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +from typing import Sequence + +from schemas.prediction.prediction_schema import SequentialSentenceClassificationSchema +from schemas.reference.reference_schema import SequentialSentenceClassificationReference + + +def _normalize_label(label: str) -> str: + return label.strip().lower() + + +def _parse_classification(classification: str, allowed_labels: set[str]) -> str: + normalized = _normalize_label(classification) + if normalized not in allowed_labels: + return "o" + return normalized + + +def _prediction_to_label_sequence( + prediction: SequentialSentenceClassificationSchema, + sentence_count: int, + allowed_labels: set[str], +) -> list[str]: + predicted_by_id: dict[int, str] = {} + + for annotation in prediction.annotations: + if annotation.text_id < 1: + continue + predicted_by_id[annotation.text_id] = _parse_classification( + annotation.category, + allowed_labels, + ) + + return [predicted_by_id.get(index + 1, "o") for index in range(sentence_count)] + + +def _filter_unwanted_labels( + golds: list[list[str]], + preds: list[list[str]], + unwanted_labels: set[str], +) -> tuple[list[list[str]], list[list[str]]]: + if not unwanted_labels: + return golds, preds + + golds_filtered: list[list[str]] = [] + preds_filtered: list[list[str]] = [] + + for gold_sequence, pred_sequence in zip(golds, preds): + filtered_gold_sequence: list[str] = [] + filtered_pred_sequence: list[str] = [] + + for gold_label, pred_label in zip(gold_sequence, pred_sequence): + if gold_label in unwanted_labels: + continue + filtered_gold_sequence.append(gold_label) + filtered_pred_sequence.append(pred_label) + + golds_filtered.append(filtered_gold_sequence) + preds_filtered.append(filtered_pred_sequence) + + return golds_filtered, preds_filtered + + +def to_bio_format(label_sequences: list[list[str]]) -> list[list[str]]: + bio_sequences: list[list[str]] = [] + + for label_sequence in label_sequences: + bio_sequence: list[str] = [] + previous_label = "o" + + for label in label_sequence: + if label == "o": + bio_sequence.append("O") + elif label != previous_label: + bio_sequence.append(f"B-{label}") + else: + bio_sequence.append(f"I-{label}") + + previous_label = label + + bio_sequences.append(bio_sequence) + + return bio_sequences + + +def build_label_sequences( + predictions: Sequence[SequentialSentenceClassificationSchema], + references: Sequence[SequentialSentenceClassificationReference], +) -> tuple[list[list[str]], list[list[str]]]: + unwanted_labels = { + _normalize_label(unwanted_label) + for reference in references + for unwanted_label in reference.unwanted_labels + if _normalize_label(unwanted_label) + } + + allowed_labels = { + _normalize_label(label) + for reference in references + for label in reference.labels + if _normalize_label(label) and _normalize_label(label) not in unwanted_labels + } + + gold_sequences: list[list[str]] = [] + pred_sequences: list[list[str]] = [] + + for prediction, reference in zip(predictions, references): + normalized_gold_labels = [_normalize_label(label) for label in reference.labels] + sentence_count = len(reference.sentences) + + gold_sequence = normalized_gold_labels + pred_sequence = _prediction_to_label_sequence( + prediction, + sentence_count, + allowed_labels, + ) + + gold_sequences.append(gold_sequence) + pred_sequences.append(pred_sequence) + + return _filter_unwanted_labels(gold_sequences, pred_sequences, unwanted_labels) diff --git a/benchmarks/src/evaluation/span_classification_artifacts.py b/benchmarks/src/evaluation/span_classification_artifacts.py new file mode 100644 index 000000000..a029ab136 --- /dev/null +++ b/benchmarks/src/evaluation/span_classification_artifacts.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Sequence + +import pandas as pd +from seqeval.metrics import classification_report + +from evaluation.artifact_base import BaseArtifactBuilder +from evaluation.span_classification_utils import ( + parse_span_reference, + spans_to_tag_ids, +) +from schemas.prediction.prediction_schema import ( + BaseAnswerSchema, + SpanClassificationSchema, +) +from schemas.reference.reference_schema import ( + BaseReferenceSchema, + SpanClassificationReference, +) + + +class SpanClassificationReportArtifacts( + BaseArtifactBuilder[SpanClassificationSchema, SpanClassificationReference] +): + def build( + self, + predictions: list[BaseAnswerSchema | None], + references: Sequence[BaseReferenceSchema], + output_dir: Path, + artifact_prefix: str, + ) -> list[Path]: + filtered_predictions, filtered_references = self.discard_none_predictions( + predictions, + references, + ) + typed_predictions = self.require_answer_schema(filtered_predictions) + typed_references = self.require_reference_schema(filtered_references) + + if len(typed_predictions) == 0: + return [] + + gold_label_sequences: list[list[str]] = [] + predicted_label_sequences: list[list[str]] = [] + + for prediction, typed_reference in zip(typed_predictions, typed_references): + tokens, gold_tag_ids, id2label, label2id = parse_span_reference( + typed_reference + ) + + predicted_tag_ids = spans_to_tag_ids( + tokens=tokens, + predicted_spans=prediction.predictions, + label2id=label2id, + ) + + gold_label_sequences.append( + [id2label[label_id] for label_id in gold_tag_ids] + ) + predicted_label_sequences.append( + [id2label[label_id] for label_id in predicted_tag_ids] + ) + + report = classification_report( + gold_label_sequences, + predicted_label_sequences, + output_dict=True, + ) + csv_path = output_dir / f"{artifact_prefix}_span_classification_report.csv" + pd.DataFrame(report).transpose().to_csv(csv_path) + return [csv_path] diff --git a/benchmarks/src/evaluation/span_classification_metrics.py b/benchmarks/src/evaluation/span_classification_metrics.py new file mode 100644 index 000000000..cd0bcc20d --- /dev/null +++ b/benchmarks/src/evaluation/span_classification_metrics.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from typing import Any, Sequence, cast + +from seqeval.metrics import accuracy_score, classification_report, f1_score + +from evaluation.metric_base import BaseMetricWrapper +from evaluation.span_classification_utils import ( + parse_span_reference, + spans_to_tag_ids, +) +from schemas.prediction.prediction_schema import ( + BaseAnswerSchema, + SpanClassificationSchema, +) +from schemas.reference.reference_schema import ( + BaseReferenceSchema, + SpanClassificationReference, +) + + +class SpanClassificationMetrics( + BaseMetricWrapper[SpanClassificationSchema, SpanClassificationReference] +): + def compute( + self, + predictions: list[BaseAnswerSchema | None], + references: Sequence[BaseReferenceSchema], + ) -> dict[str, float]: + filtered_predictions, filtered_references = self.discard_none_predictions( + predictions, + references, + ) + typed_predictions = self.require_answer_schema(filtered_predictions) + typed_references = self.require_reference_schema(filtered_references) + + if len(typed_predictions) == 0: + return { + "precision": 0.0, + "recall": 0.0, + "f1": 0.0, + "accuracy": 0.0, + } + + gold_label_sequences: list[list[str]] = [] + predicted_label_sequences: list[list[str]] = [] + + for prediction, reference in zip(typed_predictions, typed_references): + tokens, gold_tag_ids, id2label, label2id = parse_span_reference(reference) + + predicted_tag_ids = spans_to_tag_ids( + tokens=tokens, + predicted_spans=prediction.predictions, + label2id=label2id, + ) + + gold_label_sequences.append( + [id2label[label_id] for label_id in gold_tag_ids] + ) + predicted_label_sequences.append( + [id2label[label_id] for label_id in predicted_tag_ids] + ) + + accuracy = float( + accuracy_score(gold_label_sequences, predicted_label_sequences) + ) + report = classification_report( + gold_label_sequences, + predicted_label_sequences, + output_dict=True, + ) + report_dict = cast(dict[str, Any], report) + weighted_report = cast(dict[str, Any], report_dict.get("weighted avg", {})) + f1 = float( + cast( + float, + f1_score(gold_label_sequences, predicted_label_sequences), + ) + ) + + return { + "precision": float(weighted_report.get("precision", 0.0)), + "recall": float(weighted_report.get("recall", 0.0)), + "f1": f1, + "accuracy": accuracy, + } diff --git a/benchmarks/src/evaluation/span_classification_utils.py b/benchmarks/src/evaluation/span_classification_utils.py new file mode 100644 index 000000000..92bc2b266 --- /dev/null +++ b/benchmarks/src/evaluation/span_classification_utils.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import json +from typing import Any + +from schemas.prediction.prediction_schema import SpanPrediction +from schemas.reference.reference_schema import SpanClassificationReference + + +def normalize_label_name(label: str) -> str: + cleaned = label.strip() + + if cleaned.startswith("<") and cleaned.endswith(">") and len(cleaned) > 2: + cleaned = cleaned[1:-1] + + if cleaned.startswith("**") and cleaned.endswith("**") and len(cleaned) > 4: + cleaned = cleaned[2:-2] + + return cleaned.strip().lower() + + +def normalize_tokens(value: Any) -> list[str]: + if value is None: + return [] + + if hasattr(value, "tolist"): + converted = value.tolist() + if converted is not value: + return normalize_tokens(converted) + + if isinstance(value, str): + return [token for token in value.split() if token] + + if isinstance(value, (list, tuple)): + return [str(token) for token in value] + + return [str(value)] + + +def normalize_tag_ids(value: Any) -> list[int]: + if value is None: + return [] + + if hasattr(value, "tolist"): + converted = value.tolist() + if converted is not value: + return normalize_tag_ids(converted) + + if isinstance(value, str): + stripped = value.strip() + if not stripped: + return [] + + try: + decoded = json.loads(stripped) + if isinstance(decoded, list): + return [int(item) for item in decoded] + except json.JSONDecodeError: + pass + + return [int(item.strip()) for item in stripped.split(",") if item.strip()] + + if isinstance(value, (list, tuple)): + return [int(item) for item in value] + + return [int(value)] + + +def parse_span_reference( + reference: SpanClassificationReference, +) -> tuple[list[str], list[int], dict[int, str], dict[str, int]]: + tokens = normalize_tokens(reference.tokens) + tag_ids = normalize_tag_ids(reference.tag_ids) + id2label = { + int(label_id): str(label_name) + for label_id, label_name in reference.id2label.items() + } + + if 0 not in id2label: + id2label[0] = "O" + + if len(tokens) != len(tag_ids): + raise ValueError( + "Span reference tokens and tag_ids must have the same length. " + f"Got tokens={len(tokens)} and tag_ids={len(tag_ids)}." + ) + + unknown_ids = [label_id for label_id in tag_ids if label_id not in id2label] + if unknown_ids: + raise ValueError( + "Span reference contains unknown tag id(s): " + + ", ".join(str(value) for value in sorted(set(unknown_ids))) + ) + + label2id = { + normalize_label_name(label): label_id for label_id, label in id2label.items() + } + + return tokens, tag_ids, id2label, label2id + + +def spans_to_tag_ids( + tokens: list[str], + predicted_spans: list[SpanPrediction], + label2id: dict[str, int], +) -> list[int]: + predicted_tag_ids = [0] * len(tokens) + + for span in predicted_spans: + label = normalize_label_name(span.category) + text = span.text.strip() + + if not label or not text or label not in label2id: + continue + + span_tokens = text.split() + if not span_tokens: + continue + + span_length = len(span_tokens) + for start_idx in range(len(tokens)): + if start_idx + span_length > len(tokens): + break + + if tokens[start_idx : start_idx + span_length] == span_tokens: + predicted_tag_ids[start_idx : start_idx + span_length] = [ + label2id[label] + ] * span_length + + return predicted_tag_ids diff --git a/benchmarks/src/evaluation/template_filling_metrics.py b/benchmarks/src/evaluation/template_filling_metrics.py new file mode 100644 index 000000000..35602ae22 --- /dev/null +++ b/benchmarks/src/evaluation/template_filling_metrics.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +from typing import Any, Generic, Sequence, TypeVar + +import evaluate + +from evaluation.metric_base import BaseMetricWrapper +from schemas.prediction.prediction_schema import BaseAnswerSchema +from schemas.prediction.template_filling_schema import TemplateFillingMUC4AnswerSchemaV1 +from schemas.reference.reference_schema import BaseReferenceSchema, MUC4Reference + +_NO_ANSWER_MARKERS = { + "", + "none", + "not answerable", + "nicht beantwortbar", +} + + +def _normalize_slot_values(value: Any) -> list[str]: + if value is None: + return [] + + if hasattr(value, "tolist"): + converted = value.tolist() + if converted is not value: + return _normalize_slot_values(converted) + + if isinstance(value, str): + cleaned = value.strip() + if cleaned.lower() in _NO_ANSWER_MARKERS: + return [] + return [cleaned] + + if isinstance(value, (list, tuple, set)): + normalized: list[str] = [] + for item in value: + normalized.extend(_normalize_slot_values(item)) + return normalized + + cleaned = str(value).strip() + if cleaned.lower() in _NO_ANSWER_MARKERS: + return [] + return [cleaned] + + +AnswerSchemaT = TypeVar("AnswerSchemaT", bound=BaseAnswerSchema) +ReferenceSchemaT = TypeVar("ReferenceSchemaT", bound=BaseReferenceSchema) + + +class TemplateFillingMetrics( + BaseMetricWrapper[AnswerSchemaT, ReferenceSchemaT], + Generic[AnswerSchemaT, ReferenceSchemaT], +): + def __init__(self) -> None: + super().__init__() + self.metric = evaluate.load("squad_v2") + self.slots = self._resolve_slots() + + def _resolve_slots(self) -> list[str]: + answer_fields = set(self.answer_schema_cls.model_fields.keys()) + reference_fields = set(self.reference_schema_cls.model_fields.keys()) + + if answer_fields != reference_fields: + missing_in_reference = sorted(answer_fields - reference_fields) + missing_in_answer = sorted(reference_fields - answer_fields) + raise ValueError( + "Answer schema and reference schema must have exactly the same fields. " + f"Missing in reference: {missing_in_reference}. " + f"Missing in answer: {missing_in_answer}." + ) + + if not answer_fields: + raise ValueError("Template filling schema must define at least one slot.") + + return sorted(answer_fields) + + def compute( + self, + predictions: list[BaseAnswerSchema | None], + references: Sequence[BaseReferenceSchema], + ) -> dict[str, float]: + filtered_predictions, filtered_references = self.discard_none_predictions( + predictions, + references, + ) + typed_predictions = self.require_answer_schema(filtered_predictions) + typed_references = self.require_reference_schema(filtered_references) + + if len(typed_predictions) == 0: + raise ValueError("predictions and references must not be empty.") + + reference_payloads = [reference.model_dump() for reference in typed_references] + prediction_payloads = [ + prediction.get_prediction() for prediction in typed_predictions + ] + + exact_scores: list[float] = [] + f1_scores: list[float] = [] + slot_metrics: dict[str, float] = {} + + for slot in self.slots: + evaluation_predictions: list[dict[str, Any]] = [] + evaluation_references: list[dict[str, Any]] = [] + + for idx, (prediction_payload, reference_payload) in enumerate( + zip(prediction_payloads, reference_payloads) + ): + predicted_values = _normalize_slot_values( + prediction_payload.get(slot, []) + ) + reference_values = _normalize_slot_values( + reference_payload.get(slot, []) + ) + + has_answer = len(predicted_values) > 0 + evaluation_predictions.append( + { + "id": str(idx), + "prediction_text": predicted_values[0] if has_answer else "", + "no_answer_probability": 0.0 if has_answer else 1.0, + } + ) + evaluation_references.append( + { + "id": str(idx), + "answers": { + "text": reference_values, + "answer_start": [0] * len(reference_values), + }, + } + ) + + results = self.metric.compute( + predictions=evaluation_predictions, + references=evaluation_references, + ) + if results is None: + raise ValueError("squad_v2 metric returned None.") + + slot_exact = round(float(results.get("exact", 0.0)), 2) + slot_f1 = round(float(results.get("f1", 0.0)), 2) + + exact_scores.append(slot_exact) + f1_scores.append(slot_f1) + slot_metrics[f"{slot}_exact_match"] = slot_exact + slot_metrics[f"{slot}_f1"] = slot_f1 + + avg_exact = round(sum(exact_scores) / len(exact_scores), 2) + avg_f1 = round(sum(f1_scores) / len(f1_scores), 2) + + return { + "avg_exact_match": avg_exact, + "avg_f1": avg_f1, + **slot_metrics, + } + + +class TemplateFillingMUC4Metrics( + TemplateFillingMetrics[TemplateFillingMUC4AnswerSchemaV1, MUC4Reference] +): + pass diff --git a/benchmarks/src/prompts/document_classification_multi_label_v1_en.j2 b/benchmarks/src/prompts/document_classification_multi_label_v1_en.j2 new file mode 100644 index 000000000..d48573189 --- /dev/null +++ b/benchmarks/src/prompts/document_classification_multi_label_v1_en.j2 @@ -0,0 +1,20 @@ +You are a strict multi-label document classifier. + +Task: +Classify the given document into all matching categories. +Return one or more categories from the allowed list. + +Allowed categories and definitions: +{% for label in labels -%} +- {{ label }} +{% endfor %} + +Rules: +1. Return only valid JSON. +2. JSON must contain keys "reasoning" and "categories". +3. Keep "reasoning" concise. +4. "categories" must be a non-empty array of allowed categories. +5. Do not include duplicate categories. + +Document: +{{ text }} diff --git a/benchmarks/src/prompts/document_classification_single_label_v1_de.j2 b/benchmarks/src/prompts/document_classification_single_label_v1_de.j2 new file mode 100644 index 000000000..e34bd41e8 --- /dev/null +++ b/benchmarks/src/prompts/document_classification_single_label_v1_de.j2 @@ -0,0 +1,18 @@ +Du bist ein strikter Dokumentenklassifizierer. + +Aufgabe: +Klassifiziere das gegebene Dokument in genau eine der erlaubten Kategorien. + +Erlaubte Kategorien und Definitionen: +{% for label in labels -%} +- {{ label }} +{% endfor %} + +Regeln: +1. Gib nur gültiges JSON zurück. +2. Das JSON muss die Schlüssel "reasoning" und "category" enthalten. +3. Nutze das Feld "reasoning", um in 1-2 Sätzen zu analysieren, welche der erlaubten Kategorien am besten zum Dokument passt. Beende die Analyse mit einem kurzen Fazit, z.B. "Daher ist die Kategorie XYZ am besten geeignet." +4. Schlussfolgere basierend auf dem Feld "reasoning" auf die Kategorie, die am besten zum Dokument passt. Gib diese Kategorie im Feld "category" zurück. + +Dokument: +{{ text }} diff --git a/benchmarks/src/prompts/document_classification_single_label_v1_en.j2 b/benchmarks/src/prompts/document_classification_single_label_v1_en.j2 new file mode 100644 index 000000000..0d1c9484c --- /dev/null +++ b/benchmarks/src/prompts/document_classification_single_label_v1_en.j2 @@ -0,0 +1,18 @@ +You are a strict document classifier. + +Task: +Classify the given document into exactly one of the allowed categories. + +Allowed categories and definitions: +{% for label in labels -%} +- {{ label }} +{% endfor %} + +Rules: +1. Return only valid JSON. +2. JSON must contain keys "reasoning" and "category". +3. Keep "reasoning" concise. +4. "category" must be one of the allowed categories. + +Document: +{{ text }} diff --git a/benchmarks/src/prompts/document_classification_system_de.j2 b/benchmarks/src/prompts/document_classification_system_de.j2 new file mode 100644 index 000000000..95a82849e --- /dev/null +++ b/benchmarks/src/prompts/document_classification_system_de.j2 @@ -0,0 +1,2 @@ +Du bist ein System zur Unterstützung bei der Analyse großer Textmengen. +In diesem Projekt "{{ project_name }}" geht es um "{{ project_description }}". diff --git a/benchmarks/src/prompts/document_classification_system_en.j2 b/benchmarks/src/prompts/document_classification_system_en.j2 new file mode 100644 index 000000000..488592f0e --- /dev/null +++ b/benchmarks/src/prompts/document_classification_system_en.j2 @@ -0,0 +1,2 @@ +You are a system that supports the analysis of large amounts of text. +This project "{{ project_name }}" is about "{{ project_description }}". diff --git a/benchmarks/src/prompts/extractive_qa_squad1_v1_en.j2 b/benchmarks/src/prompts/extractive_qa_squad1_v1_en.j2 new file mode 100644 index 000000000..f84ab3d33 --- /dev/null +++ b/benchmarks/src/prompts/extractive_qa_squad1_v1_en.j2 @@ -0,0 +1,11 @@ +Please extract a short answer to the following question from the context. + +Context: {{ context }} + +Question: {{ question }} + +Output rules: +1. Return only valid JSON. +2. JSON must contain keys "reasoning" and "answer". +3. Keep "reasoning" concise. +4. The value in "answer" must be verbatim from the context. diff --git a/benchmarks/src/prompts/extractive_qa_squad2_v1_en.j2 b/benchmarks/src/prompts/extractive_qa_squad2_v1_en.j2 new file mode 100644 index 000000000..20491e97e --- /dev/null +++ b/benchmarks/src/prompts/extractive_qa_squad2_v1_en.j2 @@ -0,0 +1,12 @@ +Please extract a short answer to the following question from the context. + +Context: {{ context }} + +Question: {{ question }} + +Output rules: +1. Return only valid JSON. +2. JSON must contain keys "reasoning" and "answer". +3. Keep "reasoning" concise. +4. If the question cannot be answered from the context, set "answer" to "{{ no_answer_label }}". +5. The value in "answer" must be verbatim from the context, unless it is "{{ no_answer_label }}". diff --git a/benchmarks/src/prompts/extractive_qa_system_de.j2 b/benchmarks/src/prompts/extractive_qa_system_de.j2 new file mode 100644 index 000000000..e32405c83 --- /dev/null +++ b/benchmarks/src/prompts/extractive_qa_system_de.j2 @@ -0,0 +1,2 @@ +Du bist ein System zur Unterstützung bei der Analyse großer Textmengen. +Du wirst dem Nutzer helfen, alle Fragen korrekt zu beantworten. diff --git a/benchmarks/src/prompts/extractive_qa_system_en.j2 b/benchmarks/src/prompts/extractive_qa_system_en.j2 new file mode 100644 index 000000000..0dbd099ee --- /dev/null +++ b/benchmarks/src/prompts/extractive_qa_system_en.j2 @@ -0,0 +1,2 @@ +You are a system to support the analysis of large amounts of text. +You will assist the user by answering all questions correctly. diff --git a/benchmarks/src/prompts/extractive_qa_v1_de.j2 b/benchmarks/src/prompts/extractive_qa_v1_de.j2 new file mode 100644 index 000000000..138b3411e --- /dev/null +++ b/benchmarks/src/prompts/extractive_qa_v1_de.j2 @@ -0,0 +1,11 @@ +Bitte extrahiere eine kurze Antwort auf die folgende Frage aus dem Kontext. + +Kontext: {{ context }} + +Frage: {{ question }} + +Ausgaberegeln: +1. Gib nur gültiges JSON zurück. +2. Das JSON muss die Schlüssel "reasoning" und "answer" enthalten. +3. Halte "reasoning" kurz. +4. Der Wert in "answer" muss wörtlich aus dem Kontext stammen. diff --git a/benchmarks/src/prompts/sequential_sentence_classification_system_v1_en.j2 b/benchmarks/src/prompts/sequential_sentence_classification_system_v1_en.j2 new file mode 100644 index 000000000..6c37b8ff4 --- /dev/null +++ b/benchmarks/src/prompts/sequential_sentence_classification_system_v1_en.j2 @@ -0,0 +1,24 @@ +You are a professional annotator specialized in annotating {{ annotation_target }} with the help of annotation guidelines. +You strictly adhere to the guidelines and follow the desired output format. +You are a member of the project {{ project_name }} which is about {{ project_details }}. + +Annotation Guidelines: +{% for guideline in annotation_guidelines -%} +- {{ guideline }} +{% endfor %} + +Output Format: +You MUST answer in this JSON format, but the reason is optional: +{ + "annotations": [ + { + "text_id": 1, + "reason": "This text belongs to category X.", + "category": "