Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Polygon tracking feature with Cutie #8261

Closed
wants to merge 4 commits into from
Closed

[WIP] Polygon tracking feature with Cutie #8261

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6b3614d
Add button for polygon tracking
aashutoshpy Aug 6, 2024
26a6834
Add nuclio function for cutie
aashutoshpy Aug 6, 2024
744cec7
Event Listener for when canvas is drawn on.
aashutoshpy Aug 6, 2024
1834806
set shape type as rectangle if length of points array is 4
aashutoshpy Aug 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 28 additions & 3 deletions ...-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -504,7 +504,7 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
try {
const { points } = (e as CustomEvent).detail.shapes[0];
const state = new core.classes.ObjectState({
shapeType: ShapeType.RECTANGLE,
shapeType: ShapeType.POLYGON, // changed by aashutosh
objectType: ObjectType.TRACK,
source: core.enums.Source.SEMI_AUTO,
zOrder: curZOrder,
Expand Down Expand Up @@ -592,7 +592,7 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
const portals = !activeTracker ?
[] :
states
.filter((objectState) => objectState.objectType === 'track' && objectState.shapeType === 'rectangle')
.filter((objectState) => objectState.objectType === 'track' && objectState.shapeType === 'polygon') // changed by aashutosh
.map((objectState: any): React.ReactPortal | null => {
const { clientID } = objectState;
const selectorID = `#cvat-objects-sidebar-state-item-${clientID}`;
Expand Down Expand Up @@ -1046,7 +1046,32 @@ export class ToolsControlComponent extends React.PureComponent<Props, State> {
}
}}
>
Track
Track Rectangle
</Button>
</Col>
<Col>
<Button
type='primary'
loading={fetching}
className='cvat-tools-track-button'
disabled={!activeTracker || fetching || frame === jobInstance.stopFrame}
onClick={() => {
if (activeTracker && activeLabelID) {
this.setState({ mode: 'tracking' });

canvasInstance.cancel();
canvasInstance.draw({
shapeType: 'polygon',
enabled: true,
});

onInteractionStart(activeTracker, activeLabelID);
const { onSwitchToolsBlockerState } = this.props;
onSwitchToolsBlockerState({ buttonVisible: false });
}
}}
>
Track Polygon
</Button>
</Col>
</Row>
Expand Down
147 changes: 147 additions & 0 deletions serverless/pytorch/redefine/cutie/nuclio/config/eval_config.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
defaults:
- _self_
- model: base
- override hydra/job_logging: custom-no-rank.yaml

hydra:
run:
dir: ../output/${exp_id}/${dataset}
output_subdir: ${now:%Y-%m-%d_%H-%M-%S}-hydra

exp_id: default
dataset: d17-val
amp: False
weights: /opt/nuclio/cutie/cutie-base-mega.pth
output_dir: null # defaults to run_dir; specify this to override
flip_aug: False

# maximum shortest side of the input; -1 means no resizing
# With eval_vos.py, we usually just use the dataset's size (resizing done in dataloader)
# this parameter is added for the sole purpose for the GUI in the current codebase
# InferenceCore will downsize the input and restore the output to the original size if needed
# if you are using this code for some other project, you can also utilize this parameter
max_internal_size: -1

# these parameters, when set, override the dataset's default; useful for debugging
image_directory: null
mask_directory: null
json_directory: null
size: null
save_all: null
use_all_masks: null
use_long_term: null
mem_every: null

# only relevant when long_term is not enabled
max_mem_frames: 5

# only relevant when long_term is enabled
long_term:
count_usage: True
max_mem_frames: 10
min_mem_frames: 5
num_prototypes: 128
max_num_tokens: 10000
buffer_tokens: 2000

top_k: 30
stagger_updates: 5
chunk_size: -1 # number of objects to process in parallel; -1 means unlimited
save_scores: False
save_aux: False
visualize: False

datasets:
d16-val:
image_directory: ../DAVIS/2016/JPEGImages/480p
mask_directory: ../DAVIS/2016/Annotations/480p
subset: ../DAVIS/2017/trainval/ImageSets/2016/val.txt
size: 480
save_all: True
use_all_masks: False
use_long_term: False
mem_every: 5
d17-val:
image_directory: ../DAVIS/2017/trainval/JPEGImages/480p
mask_directory: ../DAVIS/2017/trainval/Annotations/480p
subset: ../DAVIS/2017/trainval/ImageSets/2017/val.txt
size: 480
save_all: True
use_all_masks: False
use_long_term: False
mem_every: 5
d17-test-dev:
image_directory: ../DAVIS/2017/test-dev/JPEGImages/480p
mask_directory: ../DAVIS/2017/test-dev/Annotations/480p
subset: ../DAVIS/2017/test-dev/ImageSets/2017/test-dev.txt
size: 480
save_all: True
use_all_masks: False
use_long_term: False
mem_every: 5
y18-val:
image_directory: ../YouTube2018/all_frames/valid_all_frames/JPEGImages
mask_directory: ../YouTube2018/valid/Annotations
json_directory: ../YouTube2018/valid/meta.json
size: 480
save_all: False
use_all_masks: True
use_long_term: False
mem_every: 5
y19-val:
image_directory: ../YouTube/all_frames/valid_all_frames/JPEGImages
mask_directory: ../YouTube/valid/Annotations
json_directory: ../YouTube/valid/meta.json
size: 480
save_all: False
use_all_masks: True
use_long_term: False
mem_every: 5
mose-val:
image_directory: ../MOSE/valid/JPEGImages
mask_directory: ../MOSE/valid/Annotations
size: 480
save_all: True
use_all_masks: False
use_long_term: False
mem_every: 5
generic:
image_directory: null
mask_directory: null
size: -1
save_all: True
use_all_masks: False
use_long_term: True
mem_every: 5
burst-val:
skip_frames: -1
image_directory: ../BURST/frames/val
json_directory: ../BURST/val/first_frame_annotations.json
size: 600
save_all: False
use_long_term: True
mem_every: 10
burst-test:
skip_frames: -1
image_directory: ../BURST/frames/test
json_directory: ../BURST/test/first_frame_annotations.json
size: 600
save_all: False
use_long_term: True
mem_every: 10
lvos-val:
image_directory: ../LVOS/valid/JPEGImages
mask_directory: ../LVOS/valid/Annotations_first_only
size: 480
save_all: False
use_all_masks: True
use_long_term: True
mem_every: 5
lvos-test:
image_directory: ../LVOS/test/JPEGImages
mask_directory: ../LVOS/test/Annotations
size: 480
save_all: False
use_all_masks: True
use_long_term: True
mem_every: 5
22 changes: 22 additions & 0 deletions serverless/pytorch/redefine/cutie/nuclio/config/hydra/job_logging/custom-no-rank.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# python logging configuration for tasks
version: 1
formatters:
simple:
format: '[%(asctime)s][%(levelname)s] - %(message)s'
datefmt: '%Y-%m-%d %H:%M:%S'
handlers:
console:
class: logging.StreamHandler
formatter: simple
stream: ext://sys.stdout
file:
class: logging.FileHandler
formatter: simple
# absolute file path
filename: ${hydra.runtime.output_dir}/${now:%Y-%m-%d_%H-%M-%S}-eval.log
mode: w
root:
level: INFO
handlers: [console, file]

disable_existing_loggers: false
Comment on lines +1 to +22
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add a new line character at the end of the file.

The file lacks a new line character at the end, which is a best practice for text files.

+
Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
# python logging configuration for tasks
version: 1
formatters:
simple:
format: '[%(asctime)s][%(levelname)s] - %(message)s'
datefmt: '%Y-%m-%d %H:%M:%S'
handlers:
console:
class: logging.StreamHandler
formatter: simple
stream: ext://sys.stdout
file:
class: logging.FileHandler
formatter: simple
# absolute file path
filename: ${hydra.runtime.output_dir}/${now:%Y-%m-%d_%H-%M-%S}-eval.log
mode: w
root:
level: INFO
handlers: [console, file]
disable_existing_loggers: false
# python logging configuration for tasks
version: 1
formatters:
simple:
format: '[%(asctime)s][%(levelname)s] - %(message)s'
datefmt: '%Y-%m-%d %H:%M:%S'
handlers:
console:
class: logging.StreamHandler
formatter: simple
stream: ext://sys.stdout
file:
class: logging.FileHandler
formatter: simple
# absolute file path
filename: ${hydra.runtime.output_dir}/${now:%Y-%m-%d_%H-%M-%S}-eval.log
mode: w
root:
level: INFO
handlers: [console, file]
disable_existing_loggers: false
Tools
yamllint

[error] 22-22: no new line character at the end of file

(new-line-at-end-of-file)

22 changes: 22 additions & 0 deletions serverless/pytorch/redefine/cutie/nuclio/config/hydra/job_logging/custom.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# python logging configuration for tasks
version: 1
formatters:
simple:
format: '[%(asctime)s][%(levelname)s][r${oc.env:LOCAL_RANK}] - %(message)s'
datefmt: '%Y-%m-%d %H:%M:%S'
handlers:
console:
class: logging.StreamHandler
formatter: simple
stream: ext://sys.stdout
file:
class: logging.FileHandler
formatter: simple
# absolute file path
filename: ${hydra.runtime.output_dir}/${now:%Y-%m-%d_%H-%M-%S}-rank${oc.env:LOCAL_RANK}.log
mode: w
root:
level: INFO
handlers: [console, file]

disable_existing_loggers: false
Comment on lines +1 to +22
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add a new line character at the end of the file.

The file lacks a new line character at the end, which is a best practice for text files.

+
Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
# python logging configuration for tasks
version: 1
formatters:
simple:
format: '[%(asctime)s][%(levelname)s][r${oc.env:LOCAL_RANK}] - %(message)s'
datefmt: '%Y-%m-%d %H:%M:%S'
handlers:
console:
class: logging.StreamHandler
formatter: simple
stream: ext://sys.stdout
file:
class: logging.FileHandler
formatter: simple
# absolute file path
filename: ${hydra.runtime.output_dir}/${now:%Y-%m-%d_%H-%M-%S}-rank${oc.env:LOCAL_RANK}.log
mode: w
root:
level: INFO
handlers: [console, file]
disable_existing_loggers: false
# python logging configuration for tasks
version: 1
formatters:
simple:
format: '[%(asctime)s][%(levelname)s][r${oc.env:LOCAL_RANK}] - %(message)s'
datefmt: '%Y-%m-%d %H:%M:%S'
handlers:
console:
class: logging.StreamHandler
formatter: simple
stream: ext://sys.stdout
file:
class: logging.FileHandler
formatter: simple
# absolute file path
filename: ${hydra.runtime.output_dir}/${now:%Y-%m-%d_%H-%M-%S}-rank${oc.env:LOCAL_RANK}.log
mode: w
root:
level: INFO
handlers: [console, file]
disable_existing_loggers: false
Tools
yamllint

[error] 22-22: no new line character at the end of file

(new-line-at-end-of-file)

58 changes: 58 additions & 0 deletions serverless/pytorch/redefine/cutie/nuclio/config/model/base.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
pixel_mean: [0.485, 0.456, 0.406]
pixel_std: [0.229, 0.224, 0.225]

pixel_dim: 256
key_dim: 64
value_dim: 256
sensory_dim: 256
embed_dim: 256

pixel_encoder:
type: resnet50
ms_dims: [1024, 512, 256]

mask_encoder:
type: resnet18
final_dim: 256

pixel_pe_scale: 32
pixel_pe_temperature: 128

object_transformer:
embed_dim: ${model.embed_dim}
ff_dim: 2048
num_heads: 8
num_blocks: 3
num_queries: 16
read_from_pixel:
input_norm: False
input_add_pe: False
add_pe_to_qkv: [True, True, False]
read_from_past:
add_pe_to_qkv: [True, True, False]
read_from_memory:
add_pe_to_qkv: [True, True, False]
read_from_query:
add_pe_to_qkv: [True, True, False]
output_norm: False
query_self_attention:
add_pe_to_qkv: [True, True, False]
pixel_self_attention:
add_pe_to_qkv: [True, True, False]

object_summarizer:
embed_dim: ${model.object_transformer.embed_dim}
num_summaries: ${model.object_transformer.num_queries}
add_pe: True

aux_loss:
sensory:
enabled: True
weight: 0.01
query:
enabled: True
weight: 0.01

mask_decoder:
# first value must equal embed_dim
up_dims: [256, 128, 128]
75 changes: 75 additions & 0 deletions serverless/pytorch/redefine/cutie/nuclio/function-gpu.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# Copyright (C) 2023-2024 CVAT.ai Corporation
#
# SPDX-License-Identifier: MIT

metadata:
name: pth-redefine-cutie
namespace: redefine
annotations:
name: Cutie
version: 1
type: tracker
spec:
help_message: The tracker brings video-object-segmentation capabilities to cvat.

spec:
description: Video object segmentation with Cutie
runtime: 'python:3.9'
handler: main:handler
eventTimeout: 30s
env:
- name: PYTHONPATH
value: /opt/nuclio/cutie

build:
image: cvat.pth.redefine.cutie:latest-gpu
baseImage: nvidia/cuda:11.8.0-devel-ubuntu20.04

directives:
preCopy:
# set NVIDIA container runtime settings
- kind: ENV
value: NVIDIA_VISIBLE_DEVICES=all
- kind: ENV
value: NVIDIA_DRIVER_CAPABILITIES=compute,utility
# disable interactive frontend
- kind: ENV
value: DEBIAN_FRONTEND=noninteractive
# set workdir
- kind: WORKDIR
value: /opt/nuclio/cutie
# install basic deps
- kind: RUN
value: apt-get update && apt-get -y install curl git python3 python3-pip ffmpeg libsm6 libxext6
# install cutie deps
- kind: RUN
value: pip3 install torch==2.3.1+cu118 torchvision==0.18.1+cu118 torchaudio==2.3.1 -f https://download.pytorch.org/whl/torch_stable.html
# install cutie code
- kind: RUN
value: pip3 install git+https://github.com/hkchengrex/Cutie.git
# download weights
- kind: RUN
value: curl -LO https://github.com/hkchengrex/Cutie/releases/download/v1.0/coco_lvis_h18_itermask.pth
# download weights
- kind: RUN
value: curl -LO https://github.com/hkchengrex/Cutie/releases/download/v1.0/cutie-base-mega.pth
# map pip3 and python3 to pip and python
- kind: RUN
value: ln -s /usr/bin/pip3 /usr/local/bin/pip && ln -s /usr/bin/python3 /usr/bin/python
triggers:
myHttpTrigger:
maxWorkers: 1
kind: 'http'
workerAvailabilityTimeoutMilliseconds: 10000
attributes:
maxRequestBodySize: 33554432 # 32MB
resources:
limits:
nvidia.com/gpu: 1

platform:
attributes:
restartPolicy:
name: always
maximumRetryCount: 3
mountMode: volume
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add a new line at the end of the file.

The file is missing a new line character at the end.

-      mountMode: volume
+
Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
mountMode: volume
mountMode: volume

Loading