Skip to content

Nested schema references #55

Open
Open
Nested schema references#55
@visch

Description

@visch
visch
opened on Mar 3, 2022

Great library! Thank you

When having nested JSON objects I'd like be able to have singer.resolve_schema_references() resolve all of my definitions. In the example_parsed file below you'll notice that the members array still has references for "$ref": "#/definitions/user" . I want user to be resolved here instead of still referenced.

Example input and output below with code.

Code:

import singer
import json
with open("example.json") as f:
  old_schema = json.load(f)
  new_schema = singer.resolve_schema_references(old_schema)
  with open("example_parsed.json", "w") as w:
    w.write(json.dumps(new_schema, indent=4))

example.json

{
    "definitions": {
        "user": {
            "type": "object",
            "properties": {
                    "id": {
                        "type": "integer"
                    },
                    "username": {
                        "type": "string"
                    },
                    "email": {
                        "type": ["null", "string"]
                    },
                    "color": {
                        "type": ["null", "string"]
                    },
                    "profilePicture": {
                        "type": ["null", "string"]
                    },
                    "initials": {
                        "type": ["null", "string"]
                    },
                    "role": {
                        "type": ["null", "integer"]
                    },
                    "custom_role": {
                        "type": ["null", "integer"]
                    },
                    "last_active": {
                        "type": ["null", "string"]
                    },
                    "date_joined": {
                        "type": ["null", "string"]
                    },
                    "date_invited": {
                        "type": ["null", "string"]
                    }
            }
        },
        "members": {
            "type": ["array", "null"],
            "items": [
                {
                    "type": "object",
                    "properties": {
                        "user": {
                            "$ref": "#/definitions/user"
                        },
                        "invited_by": {
                            "$ref": "#/definitions/user"
                        }
                    }
                } 
            ]
        },
        "status": {
            "type": "object",
            "properties": {
                    "id": {
                        "type": "string"
                    },
                    "status": {
                        "type": "string"
                    },
                    "orderindex": {
                        "type": ["null", "string"]
                    },
                    "color": {
                        "type": ["null", "string"]
                    }
            }
        },
        "feature": {
                "type": "object",
                "properties": {
                    "enabled": {
                        "type": ["boolean"]
                    },
                    "start_date": {
                        "type": ["null", "boolean"]
                    },
                    "remap_due_dates": {
                        "type": ["null", "boolean"]
                    },
                    "remap_closed_due_date": {
                        "type": ["null", "boolean"]
                    },
                    "harvest": {
                        "type": ["null", "boolean"]
                    },
                    "rollup": {
                        "type": ["null", "boolean"]
                    },
                    "per_assignee": {
                        "type": ["null", "boolean"]
                    },
                    "subtasks": {
                        "type": ["null", "boolean"]
                    },
                    "checklists": {
                        "type": ["null", "boolean"]
                    }
            }
        }
    },
    "type": "object",
    "properties": {
        "id": {
            "type": ["string"]
        },
        "name": {
            "type": ["null", "string"]
        },
        "private": {
            "type": ["null", "boolean"]
        },
        "avatar": {
            "type": ["null", "string"]
        },
        "statuses": {
            "type": ["array", "null"],
            "items": {
                "$ref": "#definitions/status"
            }
        },
        "members": {
                "$ref": "#definitions/status"
        },
        "multiple_assignees": {
            "type": ["null", "boolean"]
        },
        "features": {
            "type": "object",
            "properties": {
                "due_dates": {
                    "$ref": "#definitions/feature"
                },
                "sprints": {
                    "$ref": "#definitions/feature"
                },
                "time_tracking": {
                    "$ref": "#definitions/feature"
                },
                "points": {
                    "$ref": "#definitions/feature"
                },
                "custom_items": {
                    "$ref": "#definitions/feature"
                },
                "time_estimates": {
                    "$ref": "#definitions/feature"
                },
                "check_unresolved": {
                    "$ref": "#definitions/feature"
                },
                "zoom": {
                    "$ref": "#definitions/feature"
                },
                "milestones": {
                    "$ref": "#definitions/feature"
                },
                "remap_dependencies": {
                    "$ref": "#definitions/feature"
                },
                "dependency_warning": {
                    "$ref": "#definitions/feature"
                },
                "multiple_assignees": {
                    "$ref": "#definitions/feature"
                },
                "emails": {
                    "$ref": "#definitions/feature"
                }
            }
        },
        "archived": {
            "type": ["null", "boolean"]
        }
    }
}

example_parsed.json

{
    "definitions": {
        "user": {
            "type": "object",
            "properties": {
                "id": {
                    "type": "integer"
                },
                "username": {
                    "type": "string"
                },
                "email": {
                    "type": [
                        "null",
                        "string"
                    ]
                },
                "color": {
                    "type": [
                        "null",
                        "string"
                    ]
                },
                "profilePicture": {
                    "type": [
                        "null",
                        "string"
                    ]
                },
                "initials": {
                    "type": [
                        "null",
                        "string"
                    ]
                },
                "role": {
                    "type": [
                        "null",
                        "integer"
                    ]
                },
                "custom_role": {
                    "type": [
                        "null",
                        "integer"
                    ]
                },
                "last_active": {
                    "type": [
                        "null",
                        "string"
                    ]
                },
                "date_joined": {
                    "type": [
                        "null",
                        "string"
                    ]
                },
                "date_invited": {
                    "type": [
                        "null",
                        "string"
                    ]
                }
            }
        },
        "members": {
            "type": [
                "array",
                "null"
            ],
            "items": [
                {
                    "type": "object",
                    "properties": {
                        "user": {
                            "$ref": "#/definitions/user"
                        },
                        "invited_by": {
                            "$ref": "#/definitions/user"
                        }
                    }
                }
            ]
        },
        "status": {
            "type": "object",
            "properties": {
                "id": {
                    "type": "string"
                },
                "status": {
                    "type": "string"
                },
                "orderindex": {
                    "type": [
                        "null",
                        "string"
                    ]
                },
                "color": {
                    "type": [
                        "null",
                        "string"
                    ]
                }
            }
        },
        "feature": {
            "type": "object",
            "properties": {
                "enabled": {
                    "type": [
                        "boolean"
                    ]
                },
                "start_date": {
                    "type": [
                        "null",
                        "boolean"
                    ]
                },
                "remap_due_dates": {
                    "type": [
                        "null",
                        "boolean"
                    ]
                },
                "remap_closed_due_date": {
                    "type": [
                        "null",
                        "boolean"
                    ]
                },
                "harvest": {
                    "type": [
                        "null",
                        "boolean"
                    ]
                },
                "rollup": {
                    "type": [
                        "null",
                        "boolean"
                    ]
                },
                "per_assignee": {
                    "type": [
                        "null",
                        "boolean"
                    ]
                },
                "subtasks": {
                    "type": [
                        "null",
                        "boolean"
                    ]
                },
                "checklists": {
                    "type": [
                        "null",
                        "boolean"
                    ]
                }
            }
        }
    },
    "type": "object",
    "properties": {
        "id": {
            "type": [
                "string"
            ]
        },
        "name": {
            "type": [
                "null",
                "string"
            ]
        },
        "private": {
            "type": [
                "null",
                "boolean"
            ]
        },
        "avatar": {
            "type": [
                "null",
                "string"
            ]
        },
        "statuses": {
            "type": [
                "array",
                "null"
            ],
            "items": {
                "type": "object",
                "properties": {
                    "id": {
                        "type": "string"
                    },
                    "status": {
                        "type": "string"
                    },
                    "orderindex": {
                        "type": [
                            "null",
                            "string"
                        ]
                    },
                    "color": {
                        "type": [
                            "null",
                            "string"
                        ]
                    }
                }
            }
        },
        "members": {
            "type": "object",
            "properties": {
                "id": {
                    "type": "string"
                },
                "status": {
                    "type": "string"
                },
                "orderindex": {
                    "type": [
                        "null",
                        "string"
                    ]
                },
                "color": {
                    "type": [
                        "null",
                        "string"
                    ]
                }
            }
        },
        "multiple_assignees": {
            "type": [
                "null",
                "boolean"
            ]
        },
        "features": {
            "type": "object",
            "properties": {
                "due_dates": {
                    "type": "object",
                    "properties": {
                        "enabled": {
                            "type": [
                                "boolean"
                            ]
                        },
                        "start_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_due_dates": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_closed_due_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "harvest": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "rollup": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "per_assignee": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "subtasks": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "checklists": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        }
                    }
                },
                "sprints": {
                    "type": "object",
                    "properties": {
                        "enabled": {
                            "type": [
                                "boolean"
                            ]
                        },
                        "start_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_due_dates": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_closed_due_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "harvest": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "rollup": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "per_assignee": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "subtasks": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "checklists": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        }
                    }
                },
                "time_tracking": {
                    "type": "object",
                    "properties": {
                        "enabled": {
                            "type": [
                                "boolean"
                            ]
                        },
                        "start_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_due_dates": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_closed_due_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "harvest": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "rollup": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "per_assignee": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "subtasks": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "checklists": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        }
                    }
                },
                "points": {
                    "type": "object",
                    "properties": {
                        "enabled": {
                            "type": [
                                "boolean"
                            ]
                        },
                        "start_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_due_dates": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_closed_due_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "harvest": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "rollup": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "per_assignee": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "subtasks": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "checklists": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        }
                    }
                },
                "custom_items": {
                    "type": "object",
                    "properties": {
                        "enabled": {
                            "type": [
                                "boolean"
                            ]
                        },
                        "start_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_due_dates": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_closed_due_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "harvest": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "rollup": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "per_assignee": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "subtasks": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "checklists": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        }
                    }
                },
                "time_estimates": {
                    "type": "object",
                    "properties": {
                        "enabled": {
                            "type": [
                                "boolean"
                            ]
                        },
                        "start_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_due_dates": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_closed_due_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "harvest": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "rollup": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "per_assignee": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "subtasks": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "checklists": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        }
                    }
                },
                "check_unresolved": {
                    "type": "object",
                    "properties": {
                        "enabled": {
                            "type": [
                                "boolean"
                            ]
                        },
                        "start_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_due_dates": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_closed_due_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "harvest": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "rollup": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "per_assignee": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "subtasks": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "checklists": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        }
                    }
                },
                "zoom": {
                    "type": "object",
                    "properties": {
                        "enabled": {
                            "type": [
                                "boolean"
                            ]
                        },
                        "start_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_due_dates": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_closed_due_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "harvest": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "rollup": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "per_assignee": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "subtasks": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "checklists": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        }
                    }
                },
                "milestones": {
                    "type": "object",
                    "properties": {
                        "enabled": {
                            "type": [
                                "boolean"
                            ]
                        },
                        "start_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_due_dates": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_closed_due_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "harvest": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "rollup": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "per_assignee": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "subtasks": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "checklists": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        }
                    }
                },
                "remap_dependencies": {
                    "type": "object",
                    "properties": {
                        "enabled": {
                            "type": [
                                "boolean"
                            ]
                        },
                        "start_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_due_dates": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_closed_due_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "harvest": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "rollup": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "per_assignee": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "subtasks": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "checklists": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        }
                    }
                },
                "dependency_warning": {
                    "type": "object",
                    "properties": {
                        "enabled": {
                            "type": [
                                "boolean"
                            ]
                        },
                        "start_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_due_dates": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_closed_due_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "harvest": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "rollup": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "per_assignee": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "subtasks": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "checklists": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        }
                    }
                },
                "multiple_assignees": {
                    "type": "object",
                    "properties": {
                        "enabled": {
                            "type": [
                                "boolean"
                            ]
                        },
                        "start_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_due_dates": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_closed_due_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "harvest": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "rollup": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "per_assignee": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "subtasks": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "checklists": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        }
                    }
                },
                "emails": {
                    "type": "object",
                    "properties": {
                        "enabled": {
                            "type": [
                                "boolean"
                            ]
                        },
                        "start_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_due_dates": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "remap_closed_due_date": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "harvest": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "rollup": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "per_assignee": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "subtasks": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        },
                        "checklists": {
                            "type": [
                                "null",
                                "boolean"
                            ]
                        }
                    }
                }
            }
        },
        "archived": {
            "type": [
                "null",
                "boolean"
            ]
        }
    }
}

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Fields

    No fields configured for issues without a type.

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions