først commit for mvp

Author: tobiasmcvey

.gitignore

@@ -0,0 +1,3 @@
+data/
+.env
+poetry.lock

CODEOWNERS

@@ -0,0 +1 @@
+* @navikt/personbruker

main.py

@@ -0,0 +1,70 @@
+# %%
+import os
+import re
+
+import pandas as pd
+
+# %%
+def find_substring(substring: str, df):
+    """
+    Finn rader i en dataframe som inneholder en string
+    """
+    mask = df.applymap(
+        lambda x: substring in x.lower() if isinstance(x, str) else False
+    ).to_numpy()
+    df_results = df.loc[mask]
+    return df_results
+
+
+# %%
+def find_substring_index(substring: str, df):
+    """
+    Finn index for rader som inneholder en string i en dataframe
+    """
+    mask = df.applymap(
+        lambda x: substring in x.lower() if isinstance(x, str) else False
+    ).to_numpy()
+    df_results = df.loc[mask]
+    return df_results
+
+
+# %%
+def find_substring_regex(regex: str, df, case=False):
+    """
+    Finn rader i en dataframe der innholdet matcher regulæruttrykket
+    """
+    textlikes = df.select_dtypes(include=[object, "string"])
+    return df[
+        textlikes.apply(
+            lambda column: column.str.contains(regex, regex=True, case=case, na=False)
+        ).any(axis=1)
+    ]
+
+
+# %%
+df = pd.read_excel("data/final/merged.xlsx")
+# %%
+def finn_personer(df):
+    """
+    Finn rader som kan inneholde personopplysninger og returnér en dataframe med treff
+    """
+    dfs = []
+    df_mvh = find_substring("mvh", df)
+    df_hilsen = find_substring("hilsen", df)
+    df_epost = find_substring("@", df)
+    df_tlfnummer = find_substring_regex(r"^((0047)?|(\+47)?)[4|9]\d{7}$", df, False)
+    df_tlfnummer_space = find_substring_regex(r"^[4|9]\d{2} \d{2} \d{3}$", df, False)
+    dfs.append(df_mvh)
+    dfs.append(df_hilsen)
+    dfs.append(df_epost)
+    dfs.append(df_tlfnummer)
+    dfs.append(df_tlfnummer_space)
+    df2 = pd.concat(dfs).drop_duplicates()
+    return df2
+
+# %%
+df2 = finn_personer(df)
+len(df2)  # antall rader som inneholder navn, tlf eller epost med hilsen
+# %%
+(len(df2) / len(df))*100 # andel svar i undersøkelsen som inneholder personopplysninger
+# %%

pyproject.toml

@@ -0,0 +1,21 @@
+[tool.poetry]
+name = "dataprodukt-toppoppgaver-deling"
+version = "0.1.0"
+description = ""
+authors = ["Tobias McVey <[email protected]>"]
+license = "MIT"
+readme = "README.md"
+packages = [{include = "dataprodukt_toppoppgaver_deling"}]
+
+[tool.poetry.dependencies]
+python = "^3.10"
+
+[tool.poetry.group.dev.dependencies]
+black = "^22.10.0"
+ipykernel = "^6.17.1"
+pandas = "^1.5.1"
+openpyxl = "^3.0.10"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"