[WIP] Polygon tracking feature with Cutie

cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx

@@ -269,6 +269,7 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
         });
         canvasInstance.html().addEventListener('canvas.interacted', this.interactionListener);
         canvasInstance.html().addEventListener('canvas.canceled', this.cancelListener);
+        canvasInstance.html().addEventListener('canvas.drawn', this.interactionListener);
     }
 
     public componentDidUpdate(prevProps: Props, prevState: State): void {
@@ -333,6 +334,7 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
         onRemoveAnnotations(null);
         canvasInstance.html().removeEventListener('canvas.interacted', this.interactionListener);
         canvasInstance.html().removeEventListener('canvas.canceled', this.cancelListener);
+        canvasInstance.html().removeEventListener('canvas.drawn', this.interactionListener);
     }
 
     private contextmenuDisabler = (e: MouseEvent): void => {
@@ -503,8 +505,9 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
 
         try {
             const { points } = (e as CustomEvent).detail.shapes[0];
+            const shapeType = points.length === 4 ? ShapeType.RECTANGLE : ShapeType.POLYGON;
             const state = new core.classes.ObjectState({
-                shapeType: ShapeType.RECTANGLE,
+                shapeType, // changed by aashutosh
                 objectType: ObjectType.TRACK,
                 source: core.enums.Source.SEMI_AUTO,
                 zOrder: curZOrder,
@@ -592,7 +595,7 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
         const portals = !activeTracker ?
             [] :
             states
-                .filter((objectState) => objectState.objectType === 'track' && objectState.shapeType === 'rectangle')
+                .filter((objectState) => objectState.objectType === 'track' && objectState.shapeType === 'polygon') // changed by aashutosh
                 .map((objectState: any): React.ReactPortal | null => {
                     const { clientID } = objectState;
                     const selectorID = `#cvat-objects-sidebar-state-item-${clientID}`;
@@ -821,7 +824,9 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
                             job: jobInstance.id,
                         }) as TrackerResults;
 
-                        response.shapes = response.shapes.map(trackedRectangleMapper);
+                        if (response.shapes[0].length === 4) {
+                            response.shapes = response.shapes.map(trackedRectangleMapper);
+                        }
                         for (let i = 0; i < trackableObjects.clientIDs.length; i++) {
                             const clientID = trackableObjects.clientIDs[i];
                             const shape = response.shapes[i];
@@ -1046,7 +1051,32 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
                                 }
                             }}
                         >
-                            Track
+                            Track Rectangle
+                        </Button>
+                    </Col>
+                    <Col>
+                        <Button
+                            type='primary'
+                            loading={fetching}
+                            className='cvat-tools-track-button'
+                            disabled={!activeTracker || fetching || frame === jobInstance.stopFrame}
+                            onClick={() => {
+                                if (activeTracker && activeLabelID) {
+                                    this.setState({ mode: 'tracking' });
+
+                                    canvasInstance.cancel();
+                                    canvasInstance.draw({
+                                        shapeType: 'polygon',
+                                        enabled: true,
+                                    });
+
+                                    onInteractionStart(activeTracker, activeLabelID);
+                                    const { onSwitchToolsBlockerState } = this.props;
+                                    onSwitchToolsBlockerState({ buttonVisible: false });
+                                }
+                            }}
+                        >
+                            Track Polygon
                         </Button>
                     </Col>
                 </Row>

serverless/pytorch/redefine/cutie/nuclio/config/eval_config.yaml

@@ -0,0 +1,147 @@
+defaults:
+  - _self_
+  - model: base
+  - override hydra/job_logging: custom-no-rank.yaml
+
+hydra:
+  run:
+    dir: ../output/${exp_id}/${dataset}
+  output_subdir: ${now:%Y-%m-%d_%H-%M-%S}-hydra
+
+exp_id: default
+dataset: d17-val
+amp: False
+weights: /opt/nuclio/cutie/cutie-base-mega.pth
+output_dir: null # defaults to run_dir; specify this to override
+flip_aug: False
+
+# maximum shortest side of the input; -1 means no resizing
+# With eval_vos.py, we usually just use the dataset's size (resizing done in dataloader)
+# this parameter is added for the sole purpose for the GUI in the current codebase
+# InferenceCore will downsize the input and restore the output to the original size if needed
+# if you are using this code for some other project, you can also utilize this parameter
+max_internal_size: -1
+
+# these parameters, when set, override the dataset's default; useful for debugging
+image_directory: null
+mask_directory: null
+json_directory: null
+size: null
+save_all: null
+use_all_masks: null
+use_long_term: null
+mem_every: null
+
+# only relevant when long_term is not enabled
+max_mem_frames: 5
+
+# only relevant when long_term is enabled
+long_term:
+  count_usage: True
+  max_mem_frames: 10
+  min_mem_frames: 5
+  num_prototypes: 128
+  max_num_tokens: 10000
+  buffer_tokens: 2000
+
+top_k: 30
+stagger_updates: 5
+chunk_size: -1 # number of objects to process in parallel; -1 means unlimited
+save_scores: False
+save_aux: False
+visualize: False
+
+datasets:
+  d16-val:
+    image_directory: ../DAVIS/2016/JPEGImages/480p
+    mask_directory: ../DAVIS/2016/Annotations/480p
+    subset: ../DAVIS/2017/trainval/ImageSets/2016/val.txt
+    size: 480
+    save_all: True
+    use_all_masks: False
+    use_long_term: False
+    mem_every: 5
+  d17-val:
+    image_directory: ../DAVIS/2017/trainval/JPEGImages/480p
+    mask_directory: ../DAVIS/2017/trainval/Annotations/480p
+    subset: ../DAVIS/2017/trainval/ImageSets/2017/val.txt
+    size: 480
+    save_all: True
+    use_all_masks: False
+    use_long_term: False
+    mem_every: 5
+  d17-test-dev:
+    image_directory: ../DAVIS/2017/test-dev/JPEGImages/480p
+    mask_directory: ../DAVIS/2017/test-dev/Annotations/480p
+    subset: ../DAVIS/2017/test-dev/ImageSets/2017/test-dev.txt
+    size: 480
+    save_all: True
+    use_all_masks: False
+    use_long_term: False
+    mem_every: 5
+  y18-val:
+    image_directory: ../YouTube2018/all_frames/valid_all_frames/JPEGImages
+    mask_directory: ../YouTube2018/valid/Annotations
+    json_directory: ../YouTube2018/valid/meta.json
+    size: 480
+    save_all: False
+    use_all_masks: True
+    use_long_term: False
+    mem_every: 5
+  y19-val:
+    image_directory: ../YouTube/all_frames/valid_all_frames/JPEGImages
+    mask_directory: ../YouTube/valid/Annotations
+    json_directory: ../YouTube/valid/meta.json
+    size: 480
+    save_all: False
+    use_all_masks: True
+    use_long_term: False
+    mem_every: 5
+  mose-val:
+    image_directory: ../MOSE/valid/JPEGImages
+    mask_directory: ../MOSE/valid/Annotations
+    size: 480
+    save_all: True
+    use_all_masks: False
+    use_long_term: False
+    mem_every: 5
+  generic:
+    image_directory: null
+    mask_directory: null
+    size: -1
+    save_all: True
+    use_all_masks: False
+    use_long_term: True
+    mem_every: 5
+  burst-val:
+    skip_frames: -1
+    image_directory: ../BURST/frames/val
+    json_directory: ../BURST/val/first_frame_annotations.json
+    size: 600
+    save_all: False
+    use_long_term: True
+    mem_every: 10
+  burst-test:
+    skip_frames: -1
+    image_directory: ../BURST/frames/test
+    json_directory: ../BURST/test/first_frame_annotations.json
+    size: 600
+    save_all: False
+    use_long_term: True
+    mem_every: 10
+  lvos-val:
+    image_directory: ../LVOS/valid/JPEGImages
+    mask_directory: ../LVOS/valid/Annotations_first_only
+    size: 480
+    save_all: False
+    use_all_masks: True
+    use_long_term: True
+    mem_every: 5
+  lvos-test:
+    image_directory: ../LVOS/test/JPEGImages
+    mask_directory: ../LVOS/test/Annotations
+    size: 480
+    save_all: False
+    use_all_masks: True
+    use_long_term: True
+    mem_every: 5

serverless/pytorch/redefine/cutie/nuclio/config/hydra/job_logging/custom-no-rank.yaml

@@ -0,0 +1,22 @@
+# python logging configuration for tasks
+version: 1
+formatters:
+  simple:
+    format: '[%(asctime)s][%(levelname)s] - %(message)s'
+    datefmt: '%Y-%m-%d %H:%M:%S'
+handlers:
+  console:
+    class: logging.StreamHandler
+    formatter: simple
+    stream: ext://sys.stdout
+  file:
+    class: logging.FileHandler
+    formatter: simple
+    # absolute file path
+    filename: ${hydra.runtime.output_dir}/${now:%Y-%m-%d_%H-%M-%S}-eval.log
+    mode: w
+root:
+  level: INFO
+  handlers: [console, file]
+
+disable_existing_loggers: false

serverless/pytorch/redefine/cutie/nuclio/config/hydra/job_logging/custom.yaml

@@ -0,0 +1,22 @@
+# python logging configuration for tasks
+version: 1
+formatters:
+  simple:
+    format: '[%(asctime)s][%(levelname)s][r${oc.env:LOCAL_RANK}] - %(message)s'
+    datefmt: '%Y-%m-%d %H:%M:%S'
+handlers:
+  console:
+    class: logging.StreamHandler
+    formatter: simple
+    stream: ext://sys.stdout
+  file:
+    class: logging.FileHandler
+    formatter: simple
+    # absolute file path
+    filename: ${hydra.runtime.output_dir}/${now:%Y-%m-%d_%H-%M-%S}-rank${oc.env:LOCAL_RANK}.log
+    mode: w
+root:
+  level: INFO
+  handlers: [console, file]
+
+disable_existing_loggers: false

serverless/pytorch/redefine/cutie/nuclio/config/model/base.yaml

@@ -0,0 +1,58 @@
+pixel_mean: [0.485, 0.456, 0.406]
+pixel_std: [0.229, 0.224, 0.225]
+
+pixel_dim: 256
+key_dim: 64
+value_dim: 256
+sensory_dim: 256
+embed_dim: 256
+
+pixel_encoder:
+  type: resnet50
+  ms_dims: [1024, 512, 256]
+
+mask_encoder:
+  type: resnet18
+  final_dim: 256
+
+pixel_pe_scale: 32
+pixel_pe_temperature: 128
+
+object_transformer:
+  embed_dim: ${model.embed_dim}
+  ff_dim: 2048
+  num_heads: 8
+  num_blocks: 3
+  num_queries: 16
+  read_from_pixel:
+    input_norm: False
+    input_add_pe: False
+    add_pe_to_qkv: [True, True, False]
+  read_from_past:
+    add_pe_to_qkv: [True, True, False]
+  read_from_memory:
+    add_pe_to_qkv: [True, True, False]
+  read_from_query:
+    add_pe_to_qkv: [True, True, False]
+    output_norm: False
+  query_self_attention:
+    add_pe_to_qkv: [True, True, False]
+  pixel_self_attention:
+    add_pe_to_qkv: [True, True, False]
+
+object_summarizer:
+  embed_dim: ${model.object_transformer.embed_dim}
+  num_summaries: ${model.object_transformer.num_queries}
+  add_pe: True
+
+aux_loss:
+  sensory:
+    enabled: True
+    weight: 0.01
+  query:
+    enabled: True
+    weight: 0.01
+
+mask_decoder:
+  # first value must equal embed_dim
+  up_dims: [256, 128, 128]