Skip to content

Commit 0270919

Browse files
RICHARD-QuentinRICHARD-Quentin
committed
Mise à jour de la version de DataWeaver dans requirements.txt
1 parent 8c8ad6d commit 0270919

File tree

3 files changed

+94
-98
lines changed

3 files changed

+94
-98
lines changed

‎config.yml‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
{
2+
"taiga_etd.json": {
3+
"mapping": {
4+
"inetOrgPerson.cn": "nom",
5+
"inetOrgPerson.sn": "prenom",
6+
"inetOrgPerson.givenName": "prenom",
7+
"additionalFields.attributes.supann.supannPrenomsEtatCivil": "prenom",
8+
"inetOrgPerson.uid": "id_coord",
9+
"additionalFields.attributes.supann.supannRefId": "id_coord",
10+
"inetOrgPerson.mail": "email1",
11+
"inetOrgPerson.mobile": "tel_mob",
12+
"inetOrgPerson.postalAddress": "adresse",
13+
"inetOrgPerson.userPassword": "mot_de_passe_ldap",
14+
"additionalFields.attributes.supann.supanncivilite": "civilite",
15+
"additionalFields.attributes.supann.supannOIDCGenre": "civilite",
16+
"additionalFields.attributes.supann.supannNomdeNaissance": "nom_marital",
17+
"additionalFields.attributes.supann.supannOIDCDatedeNaissance": "nss_date",
18+
"additionalFields.attributes.supann.supannCodeINSEEPaysDeNaissance": "nns_pays",
19+
"additionalFields.attributes.supann.supannCodeINSEEVilleDeNaissance": "nss_ville",
20+
"additionalFields.attributes.supann.supannAutreMail": "email2"
21+
},
22+
"additionalFields": {
23+
"additionalFields.objectClasses": ["supann"],
24+
"additionalFields.attributes.supann.supannTypeEntiteAffectation": "etd",
25+
"state": -1
26+
}
27+
},
28+
"taiga_adm.json": {
29+
"mapping": {
30+
"inetOrgPerson.cn": "nom",
31+
"inetOrgPerson.sn": "prenom",
32+
"inetOrgPerson.givenName": "prenom",
33+
"additionalFields.attributes.supann.supannPrenomsEtatCivil": "prenom",
34+
"inetOrgPerson.uid": "id_coord",
35+
"additionalFields.attributes.supann.supannEmpId": "id_coord",
36+
"inetOrgPerson.mail": "email1",
37+
"inetOrgPerson.mobile": "tel_mob",
38+
"inetOrgPerson.postalAddress": "adresse",
39+
"inetOrgPerson.userPassword": "mot_de_passe_ldap",
40+
"additionalFields.attributes.supann.supanncivilite": "civilite",
41+
"additionalFields.attributes.supann.supannOIDCGenre": "civilite",
42+
"additionalFields.attributes.supann.supannNomdeNaissance": "nom_marital",
43+
"additionalFields.attributes.supann.supannOIDCDatedeNaissance": "nss_date",
44+
"additionalFields.attributes.supann.supannCodeINSEEPaysDeNaissance": "nns_pays",
45+
"additionalFields.attributes.supann.supannCodeINSEEVilleDeNaissance": "nss_ville",
46+
"additionalFields.attributes.supann.supannAutreMail": "email2"
47+
},
48+
"additionalFields": {
49+
"additionalFields.objectClasses": ["supann"],
50+
"additionalFields.attributes.supann.supannTypeEntiteAffectation": "adm",
51+
"state": -1
52+
}
53+
},
54+
"taiga_esn.json": {
55+
"mapping": {
56+
"inetOrgPerson.cn": "nom",
57+
"inetOrgPerson.sn": "prenom",
58+
"inetOrgPerson.givenName": "prenom",
59+
"additionalFields.attributes.supann.supannPrenomsEtatCivil": "prenom",
60+
"inetOrgPerson.uid": "id_coord",
61+
"additionalFields.attributes.supann.supannEmpId": "id_coord",
62+
"inetOrgPerson.mail": "email1",
63+
"inetOrgPerson.mobile": "tel_mob",
64+
"inetOrgPerson.postalAddress": "adresse",
65+
"inetOrgPerson.userPassword": "mot_de_passe_ldap",
66+
"additionalFields.attributes.supann.supanncivilite": "civilite",
67+
"additionalFields.attributes.supann.supannOIDCGenre": "civilite",
68+
"additionalFields.attributes.supann.supannNomdeNaissance": "nom_marital",
69+
"additionalFields.attributes.supann.supannOIDCDatedeNaissance": "nss_date",
70+
"additionalFields.attributes.supann.supannCodeINSEEPaysDeNaissance": "nns_pays",
71+
"additionalFields.attributes.supann.supannCodeINSEEVilleDeNaissance": "nss_ville",
72+
"additionalFields.attributes.supann.supannAutreMail": "email2"
73+
},
74+
"additionalFields": {
75+
"additionalFields.objectClasses": ["supann"],
76+
"additionalFields.attributes.supann.supannTypeEntiteAffectation": "esn",
77+
"state": -1
78+
}
79+
}
80+
}

‎requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
aiohttp==3.9.3
2-
DataWeaver==0.3.2
2+
DataWeaver==1.0.2
33
python-dotenv==1.0.1
44
Requests==2.31.0
55
urllib3==1.26.18

‎src/import_ind.py‎

Lines changed: 13 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -4,100 +4,11 @@
44
from data_weaver import weave_entries, weave_entry
55
import aiohttp
66
import dotenv
7+
import yaml
78

89
dotenv.load_dotenv()
910
sesame_api_baseurl = os.getenv('SESAME_API_BASEURL')
1011

11-
configs = {
12-
"taiga_etd.json": {
13-
"mapping": {
14-
"nom": "inetOrgPerson.cn",
15-
"prenom": ["inetOrgPerson.sn", "inetOrgPerson.givenName", "additionalFields.attributes.supann.supannPrenomsEtatCivil"],
16-
"id_coord": ["inetOrgPerson.uid", "additionalFields.attributes.supann.supannRefId"],
17-
"email1": "inetOrgPerson.mail",
18-
"tel_mob": "inetOrgPerson.mobile",
19-
"adresse": "inetOrgPerson.postalAddress",
20-
"mot_de_passe_ldap": "inetOrgPerson.userPassword",
21-
"civilite": ["additionalFields.attributes.supann.supanncivilite", "additionalFields.attributes.supann.supannOIDCGenre"],
22-
"nom_marital": "additionalFields.attributes.supann.supannNomdeNaissance",
23-
"nss_date": "additionalFields.attributes.supann.supannOIDCDatedeNaissance",
24-
"nns_pays": "additionalFields.attributes.supann.supannCodeINSEEPaysDeNaissance",
25-
"nss_ville": "additionalFields.attributes.supann.supannCodeINSEEVilleDeNaissance",
26-
"email2": "additionalFields.attributes.supann.supannAutreMail",
27-
#"sesame": "additionalFields.attributes.supann.supannListeRouge",
28-
#"outil de messagerie ou sesame ?": "additionalFields.attributes.mailForwardingAddress",
29-
#"taiga + sesame": "additionalFields.attributes.supann.supannMailPerso",
30-
#"calculé": "additionalFields.attributes.supann.supannRessourceEtatDate",
31-
#"saisie": "additionalFields.attributes.supann.supannEntiteAffectationPrincipale"
32-
},
33-
"additionalFields": {
34-
"additionalFields.objectClasses": ["supann"],
35-
"additionalFields.attributes.supann.supannTypeEntiteAffectation": "etd",
36-
"state": -1,
37-
}
38-
},
39-
"taiga_adm.json": {
40-
"mapping": {
41-
"nom": "inetOrgPerson.cn",
42-
"prenom": ["inetOrgPerson.sn", "inetOrgPerson.givenName", "additionalFields.attributes.supann.supannPrenomsEtatCivil"],
43-
"id_coord": ["inetOrgPerson.uid", "additionalFields.attributes.supann.supannEmpId"],
44-
"email1": "inetOrgPerson.mail",
45-
"tel_mob": "inetOrgPerson.mobile",
46-
"adresse": "inetOrgPerson.postalAddress",
47-
"mot_de_passe_ldap": "inetOrgPerson.userPassword",
48-
"civilite": ["additionalFields.attributes.supann.supanncivilite", "additionalFields.attributes.supann.supannOIDCGenre"],
49-
"nom_marital": "additionalFields.attributes.supann.supannNomdeNaissance",
50-
"nss_date": "additionalFields.attributes.supann.supannOIDCDatedeNaissance",
51-
"nns_pays": "additionalFields.attributes.supann.supannCodeINSEEPaysDeNaissance",
52-
"nss_ville": "additionalFields.attributes.supann.supannCodeINSEEVilleDeNaissance",
53-
"email2": "additionalFields.attributes.supann.supannAutreMail",
54-
#"sesame": "additionalFields.attributes.supann.supannListeRouge",
55-
#"outil de messagerie ou sesame ?": "additionalFields.attributes.mailForwardingAddress",
56-
#"taiga + sesame": "additionalFields.attributes.supann.supannMailPerso",
57-
#"calculé": "additionalFields.attributes.supann.supannRoleGenerique",
58-
#"saisie": "additionalFields.attributes.supann.supannParrainDN",
59-
#"calcué": "additionalFields.attributes.supann.supannTypeEntiteAffectation",
60-
#"": "additionalFields.attributes.supann.supannActivite",
61-
#"taiga, sesame ou fixe": "additionalFields.attributes.supann.supannEmpDateFin"
62-
},
63-
"additionalFields": {
64-
"additionalFields.objectClasses": ["supann"],
65-
"additionalFields.attributes.supann.supannTypeEntiteAffectation": "adm",
66-
"state": -1,
67-
}
68-
},
69-
"taiga_esn.json": {
70-
"mapping": {
71-
"nom": "inetOrgPerson.cn",
72-
"prenom": ["inetOrgPerson.sn", "inetOrgPerson.givenName", "additionalFields.attributes.supann.supannPrenomsEtatCivil"],
73-
"id_coord": ["inetOrgPerson.uid", "additionalFields.attributes.supann.supannEmpId"],
74-
"email1": "inetOrgPerson.mail",
75-
"tel_mob": "inetOrgPerson.mobile",
76-
"adresse": "inetOrgPerson.postalAddress",
77-
"mot_de_passe_ldap": "inetOrgPerson.userPassword",
78-
"civilite": ["additionalFields.attributes.supann.supanncivilite", "additionalFields.attributes.supann.supannOIDCGenre"],
79-
"nom_marital": "additionalFields.attributes.supann.supannNomdeNaissance",
80-
"nss_date": "additionalFields.attributes.supann.supannOIDCDatedeNaissance",
81-
"nns_pays": "additionalFields.attributes.supann.supannCodeINSEEPaysDeNaissance",
82-
"nss_ville": "additionalFields.attributes.supann.supannCodeINSEEVilleDeNaissance",
83-
"email2": "additionalFields.attributes.supann.supannAutreMail",
84-
#"sesame": "additionalFields.attributes.supann.supannListeRouge",
85-
#"outil de messagerie ou sesame ?": "additionalFields.attributes.mailForwardingAddress",
86-
#"taiga + sesame": "additionalFields.attributes.supann.supannMailPerso",
87-
#"calculé": "additionalFields.attributes.supann.supannRoleGenerique",
88-
#"saisie": "additionalFields.attributes.supann.supannParrainDN",
89-
#"calcué": "additionalFields.attributes.supann.supannTypeEntiteAffectation",
90-
#"": "additionalFields.attributes.supann.supannActivite",
91-
#"taiga, sesame ou fixe": "additionalFields.attributes.supann.supannEmpDateFin"
92-
},
93-
"additionalFields": {
94-
"additionalFields.objectClasses": ["supann"],
95-
"additionalFields.attributes.supann.supannTypeEntiteAffectation": "esn",
96-
"state": -1,
97-
}
98-
},
99-
}
100-
10112
async def gather_with_concurrency(n, tasks):
10213
semaphore = asyncio.Semaphore(n)
10314

@@ -125,7 +36,7 @@ async def get_data(data, config):
12536
result.append(treated)
12637
return result
12738

128-
39+
12940

13041
async def process_data(data, config, file, session):
13142
print(f"Processing {file}")
@@ -136,15 +47,20 @@ async def process_data(data, config, file, session):
13647
await gather_with_concurrency(25, tasks)
13748
print(f"Processed {file}")
13849

50+
async def load_config():
51+
with open('./config.yml', 'r', encoding='utf-8') as fichier:
52+
return yaml.load(fichier, Loader=yaml.FullLoader)
53+
13954
async def import_ind():
55+
configs = await load_config()
14056
cache_files = os.listdir('./cache')
14157
datas = {}
14258
for file in cache_files:
143-
with open(f'./cache/{file}', 'r', encoding='utf-8') as fichier:
144-
datas[file] = json.load(fichier).get('data')
59+
if file in configs.keys():
60+
print(f"Loading {file}, keys: {configs.keys()}")
61+
with open(f'./cache/{file}', 'r', encoding='utf-8') as fichier:
62+
datas[file] = json.load(fichier).get('data')
14563

14664
async with aiohttp.ClientSession() as session:
147-
# for file in cache_files:
148-
# await process_data(datas[file], configs[file], file, session)
149-
tasks = [process_data(datas[file], configs[file], file, session) for file in cache_files]
150-
await gather_with_concurrency(10, tasks)
65+
tasks = [process_data(datas[file], configs[file], file, session) for file in cache_files if file in configs.keys()]
66+
await gather_with_concurrency(10, tasks)

0 commit comments

Comments
 (0)