Skip to content

Commit f6c0000

Browse files
tobiascadeetobiascadee
committed
fix schemas for history and query
fields with a `.` do not work, replacing them with underscores.
1 parent 402c967 commit f6c0000

File tree

5 files changed

+108
-92
lines changed

5 files changed

+108
-92
lines changed

Diff for: ‎meltano.yml

+4
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,10 @@ plugins:
4545
client_secret: $TAP_LOOKER_CLIENT_SECRET
4646
filter_models: [phoenix]
4747
filter_explores: [tickets]
48+
49+
select:
50+
- "query.*"
51+
- "history.*"
4852

4953
loaders:
5054
- name: target-jsonl

Diff for: ‎tap_looker/client.py

+18-6
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,11 @@ def convert_to_dict(self, obj: object) -> dict:
5454
class LookerSystemActivityStream(LookerStream):
5555
def __init__(self, tap: Tap):
5656
super().__init__(tap)
57-
self.replication_key = f"{self.name}.created_time"
58-
self.primary_keys = [f"{self.name}.id"]
57+
self.replication_key = f"{self.name}_created_time"
58+
self.primary_keys = [f"{self.name}_id"]
59+
60+
def replace_prefix(self, field: str) -> str:
61+
return field.replace(f"{self.name}_", f"{self.name}.")
5962

6063
def get_records(
6164
self,
@@ -73,22 +76,31 @@ def get_records(
7376
Raises:
7477
NotImplementedError: If the implementation is TODO
7578
"""
76-
result_count: int = ROW_LIMIT - 1
79+
result_count: int = ROW_LIMIT
7780
result_max_date: datetime = self.get_starting_timestamp(context)
78-
while result_count != ROW_LIMIT:
81+
fields = [
82+
self.replace_prefix(field) for field in list(self.schema["properties"])
83+
]
84+
while result_count == ROW_LIMIT:
7985
response = self.sdk.run_inline_query(
8086
result_format="json",
8187
body=models.WriteQuery(
8288
model="system__activity",
8389
view=self.name,
84-
fields=list(self.schema["properties"].keys()),
90+
fields=fields,
8591
filters={
86-
self.replication_key: f"after {result_max_date.strftime('%Y-%m-%d %H:%M:%S')}",
92+
self.replace_prefix(
93+
self.replication_key
94+
): f"after {result_max_date.strftime('%Y-%m-%d %H:%M:%S')}",
8795
},
8896
),
8997
limit=ROW_LIMIT,
9098
)
9199
response_results = json.loads(response)
100+
response_results = [
101+
{key.replace(".", "_"): value for key, value in response_result.items()}
102+
for response_result in response_results
103+
]
92104
result_count = len(response_results)
93105
result_max_date = datetime.strptime( # noqa: DTZ007
94106
response_results[-1][self.replication_key],

Diff for: ‎tap_looker/schemas.py

+43-43
Original file line numberDiff line numberDiff line change
@@ -15,53 +15,53 @@
1515
)
1616

1717
query_schema = PropertiesList(
18-
Property("query.id", IntegerType),
19-
Property("query.created_time", DateTimeType),
20-
Property("query.view", StringType),
21-
Property("query.count_of_dynamic_fields", IntegerType),
22-
Property("query.dynamic_fields", StringType),
23-
Property("query.pivots", StringType),
24-
Property("query.hash", StringType),
25-
Property("query.fields", StringType),
26-
Property("query.filters", StringType),
27-
Property("query.model", StringType),
28-
Property("query.link", StringType),
29-
Property("query.query_timezone", StringType),
30-
Property("query.slug", StringType),
31-
Property("query.column_limit", StringType),
32-
Property("query.row_total", StringType),
33-
Property("query.sorts", StringType),
34-
Property("query.total", StringType),
35-
Property("query.limit", IntegerType),
18+
Property("query_id", IntegerType),
19+
Property("query_created_time", DateTimeType),
20+
Property("query_view", StringType),
21+
Property("query_count_of_dynamic_fields", IntegerType),
22+
Property("query_dynamic_fields", StringType),
23+
Property("query_pivots", StringType),
24+
Property("query_hash", StringType),
25+
Property("query_fields", StringType),
26+
Property("query_filters", StringType),
27+
Property("query_model", StringType),
28+
Property("query_link", StringType),
29+
Property("query_query_timezone", StringType),
30+
Property("query_slug", StringType),
31+
Property("query_column_limit", StringType),
32+
Property("query_row_total", StringType),
33+
Property("query_sorts", StringType),
34+
Property("query_total", StringType),
35+
Property("query_limit", IntegerType),
3636
).to_dict()
3737

3838

3939
history_schema = PropertiesList(
40-
Property("history.id", IntegerType),
41-
Property("history.cache", StringType),
42-
Property("history.cache_key", StringType),
43-
Property("history.result_source", StringType),
44-
Property("history.completed_time", DateTimeType),
45-
Property("history.connection_id", StringType),
46-
Property("history.connection_name", StringType),
47-
Property("history.dialect", StringType),
48-
Property("history.created_time", DateTimeType),
49-
Property("history.real_dash_id", StringType),
50-
Property("history.dashboard_session", StringType),
51-
Property("history.message", StringType),
52-
Property("history.most_recent_run_at_time", DateTimeType),
53-
Property("history.most_recent_length", NumberType),
54-
Property("history.is_single_query", StringType),
55-
Property("history.is_user_dashboard", StringType),
56-
Property("history.rebuild_pdts", StringType),
57-
Property("history.render_key", StringType),
58-
Property("history.result_format", StringType),
59-
Property("history.runtime", NumberType),
60-
Property("history.slug", StringType),
61-
Property("history.source", StringType),
62-
Property("history.status", StringType),
63-
Property("history.query_id", IntegerType),
64-
Property("history.workspace_id", StringType),
40+
Property("history_id", IntegerType),
41+
Property("history_cache", StringType),
42+
Property("history_cache_key", StringType),
43+
Property("history_result_source", StringType),
44+
Property("history_completed_time", DateTimeType),
45+
Property("history_connection_id", StringType),
46+
Property("history_connection_name", StringType),
47+
Property("history_dialect", StringType),
48+
Property("history_created_time", DateTimeType),
49+
Property("history_real_dash_id", StringType),
50+
Property("history_dashboard_session", StringType),
51+
Property("history_message", StringType),
52+
Property("history_most_recent_run_at_time", DateTimeType),
53+
Property("history_most_recent_length", NumberType),
54+
Property("history_is_single_query", StringType),
55+
Property("history_is_user_dashboard", StringType),
56+
Property("history_rebuild_pdts", StringType),
57+
Property("history_render_key", StringType),
58+
Property("history_result_format", StringType),
59+
Property("history_runtime", NumberType),
60+
Property("history_slug", StringType),
61+
Property("history_source", StringType),
62+
Property("history_status", StringType),
63+
Property("history_query_id", IntegerType),
64+
Property("history_workspace_id", StringType),
6565
).to_dict()
6666

6767
models_schema = PropertiesList(

Diff for: ‎tap_looker/schemas/history.json

+25-25
Original file line numberDiff line numberDiff line change
@@ -1,154 +1,154 @@
11
{
22
"type": "object",
33
"properties": {
4-
"history.id": {
4+
"history_id": {
55
"type": [
66
"integer",
77
"null"
88
]
99
},
10-
"history.cache": {
10+
"history_cache": {
1111
"type": [
1212
"string",
1313
"null"
1414
]
1515
},
16-
"history.cache_key": {
16+
"history_cache_key": {
1717
"type": [
1818
"string",
1919
"null"
2020
]
2121
},
22-
"history.result_source": {
22+
"history_result_source": {
2323
"type": [
2424
"string",
2525
"null"
2626
]
2727
},
28-
"history.completed_time": {
28+
"history_completed_time": {
2929
"type": [
3030
"string",
3131
"null"
3232
],
3333
"format": "date-time"
3434
},
35-
"history.connection_id": {
35+
"history_connection_id": {
3636
"type": [
3737
"string",
3838
"null"
3939
]
4040
},
41-
"history.connection_name": {
41+
"history_connection_name": {
4242
"type": [
4343
"string",
4444
"null"
4545
]
4646
},
47-
"history.dialect": {
47+
"history_dialect": {
4848
"type": [
4949
"string",
5050
"null"
5151
]
5252
},
53-
"history.created_time": {
53+
"history_created_time": {
5454
"type": [
5555
"string",
5656
"null"
5757
],
5858
"format": "date-time"
5959
},
60-
"history.real_dash_id": {
60+
"history_real_dash_id": {
6161
"type": [
6262
"string",
6363
"null"
6464
]
6565
},
66-
"history.dashboard_session": {
66+
"history_dashboard_session": {
6767
"type": [
6868
"string",
6969
"null"
7070
]
7171
},
72-
"history.message": {
72+
"history_message": {
7373
"type": [
7474
"string",
7575
"null"
7676
]
7777
},
78-
"history.most_recent_run_at_time": {
78+
"history_most_recent_run_at_time": {
7979
"type": [
8080
"string",
8181
"null"
8282
],
8383
"format": "date-time"
8484
},
85-
"history.most_recent_length": {
85+
"history_most_recent_length": {
8686
"type": [
8787
"number",
8888
"null"
8989
]
9090
},
91-
"history.is_single_query": {
91+
"history_is_single_query": {
9292
"type": [
9393
"string",
9494
"null"
9595
]
9696
},
97-
"history.is_user_dashboard": {
97+
"history_is_user_dashboard": {
9898
"type": [
9999
"string",
100100
"null"
101101
]
102102
},
103-
"history.rebuild_pdts": {
103+
"history_rebuild_pdts": {
104104
"type": [
105105
"string",
106106
"null"
107107
]
108108
},
109-
"history.render_key": {
109+
"history_render_key": {
110110
"type": [
111111
"string",
112112
"null"
113113
]
114114
},
115-
"history.result_format": {
115+
"history_result_format": {
116116
"type": [
117117
"string",
118118
"null"
119119
]
120120
},
121-
"history.runtime": {
121+
"history_runtime": {
122122
"type": [
123123
"number",
124124
"null"
125125
]
126126
},
127-
"history.slug": {
127+
"history_slug": {
128128
"type": [
129129
"string",
130130
"null"
131131
]
132132
},
133-
"history.source": {
133+
"history_source": {
134134
"type": [
135135
"string",
136136
"null"
137137
]
138138
},
139-
"history.status": {
139+
"history_status": {
140140
"type": [
141141
"string",
142142
"null"
143143
]
144144
},
145-
"history.query_id": {
145+
"history_query_id": {
146146
"type": [
147147
"integer",
148148
"null"
149149
]
150150
},
151-
"history.workspace_id": {
151+
"history_workspace_id": {
152152
"type": [
153153
"string",
154154
"null"

0 commit comments

Comments
 (0)