Skip to content

Commit 4c43788

Browse files
tangledbytestangledbytes
committed
add support for reserved bucket tags
Signed-off-by: Utkarsh Srivastava <[email protected]> fix put_bucket_tagging test - pass dummy objectsdk Signed-off-by: Utkarsh Srivastava <[email protected]> add support for events, race safe tag manipulation and --merge_tag flag Signed-off-by: Utkarsh Srivastava <[email protected]> fix lint issues Signed-off-by: Utkarsh Srivastava <[email protected]> clarify the reserved tags config comment Signed-off-by: Utkarsh Srivastava <[email protected]> add docs for the CLI changes Signed-off-by: Utkarsh Srivastava <[email protected]> address PR comments Signed-off-by: Utkarsh Srivastava <[email protected]> add account name to the bucket create event Signed-off-by: Utkarsh Srivastava <[email protected]>
1 parent 9ce0630 commit 4c43788

File tree

11 files changed

+315
-34
lines changed

11 files changed

+315
-34
lines changed

‎config.js

+27
Original file line numberDiff line numberDiff line change
@@ -954,6 +954,33 @@ config.NSFS_GLACIER_FORCE_EXPIRE_ON_GET = false;
954954
// interval
955955
config.NSFS_GLACIER_MIGRATE_LOG_THRESHOLD = 50 * 1024;
956956

957+
/**
958+
* NSFS_GLACIER_RESERVED_BUCKET_TAGS defines an object of bucket tags which will be reserved
959+
* by the system and PUT operations for them via S3 API would be limited - as in they would be
960+
* mutable only if specified and only under certain conditions.
961+
*
962+
* @type {Record<string, {
963+
* schema: Record<any, any> & { $id: string },
964+
* immutable: true | false | 'if-data',
965+
* default: any,
966+
* event: boolean
967+
* }>}
968+
*
969+
* @example
970+
* {
971+
'deep-archive-copies': {
972+
schema: {
973+
$id: 'deep-archive-copies-schema-v0',
974+
enum: ['1', '2']
975+
}, // JSON Schema
976+
immutable: 'if-data',
977+
default: '1',
978+
event: true
979+
}
980+
* }
981+
*/
982+
config.NSFS_GLACIER_RESERVED_BUCKET_TAGS = {};
983+
957984
// anonymous account name
958985
config.ANONYMOUS_ACCOUNT_NAME = 'anonymous';
959986

‎docs/NooBaaNonContainerized/Events.md

+10-2
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,10 @@ The following list includes events that indicate on a normal / successful operat
3232
- Description: NooBaa account was deleted successfully using NooBaa CLI.
3333

3434
#### 4. `noobaa_bucket_created`
35-
- Arguments: `bucket_name`
35+
- Arguments:
36+
- `bucket_name`
37+
- `account_name`
38+
- `<tag_value>` (if `event` is `true` for the reserved tag)
3639
- Description: NooBaa bucket was created successfully using NooBaa CLI or S3.
3740

3841
#### 5. `noobaa_bucket_deleted`
@@ -43,6 +46,11 @@ The following list includes events that indicate on a normal / successful operat
4346
- Arguments: `whitelist_ips`
4447
- Description: Whitelist Server IPs updated successfully using NooBaa CLI.
4548

49+
#### 7. `noobaa_bucket_reserved_tag_modified`
50+
- Arguments:
51+
- `bucket_name`
52+
- `<tag_value>` (if `event` is `true` for the reserved tag)
53+
- Description: NooBaa bucket reserved tag was modified successfully using NooBaa CLI or S3.
4654

4755
### Error Indicating Events
4856

@@ -219,4 +227,4 @@ The following list includes events that indicate on some sort of malfunction or
219227
- Reasons:
220228
- Free space in notification log dir FS is below threshold.
221229
- Resolutions:
222-
- Free up space is FS.
230+
- Free up space is FS.

‎docs/NooBaaNonContainerized/NooBaaCLI.md

+7
Original file line numberDiff line numberDiff line change
@@ -376,6 +376,13 @@ noobaa-cli bucket update --name <bucket_name> [--new_name] [--owner]
376376
- Type: Boolean
377377
- Description: Set the bucket to force md5 ETag calculation. Unset with ''.
378378

379+
- `tag`
380+
- Type: String
381+
- Description: Set the bucket tags, type is a string of valid JSON. Behaviour is similar to `put-bucket-tagging` S3 API.
382+
383+
- `merge_tag`
384+
- Type: String
385+
- Description: Merge the bucket tags with previous bucket tags, type is a string of valid JSON.
379386

380387
### Bucket Status
381388

‎src/cmd/manage_nsfs.js

+106-20
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ const { throw_cli_error, get_bucket_owner_account_by_name,
3939
const manage_nsfs_validations = require('../manage_nsfs/manage_nsfs_validations');
4040
const nc_mkm = require('../manage_nsfs/nc_master_key_manager').get_instance();
4141
const notifications_util = require('../util/notifications_util');
42+
const BucketSpaceFS = require('../sdk/bucketspace_fs');
43+
const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent;
4244

4345
///////////////
4446
//// GENERAL //
@@ -135,7 +137,6 @@ async function fetch_bucket_data(action, user_input) {
135137
force_md5_etag: user_input.force_md5_etag === undefined || user_input.force_md5_etag === '' ? user_input.force_md5_etag : get_boolean_or_string_value(user_input.force_md5_etag),
136138
notifications: user_input.notifications
137139
};
138-
139140
if (user_input.bucket_policy !== undefined) {
140141
if (typeof user_input.bucket_policy === 'string') {
141142
// bucket_policy deletion specified with empty string ''
@@ -154,6 +155,27 @@ async function fetch_bucket_data(action, user_input) {
154155
data = await merge_new_and_existing_config_data(data);
155156
}
156157

158+
if ((action === ACTIONS.UPDATE && user_input.tag) || (action === ACTIONS.ADD)) {
159+
const tags = JSON.parse(user_input.tag || '[]');
160+
data.tag = BucketSpaceFS._merge_reserved_tags(
161+
data.tag || BucketSpaceFS._default_bucket_tags(),
162+
tags,
163+
action === ACTIONS.ADD ? true : await _is_bucket_empty(data),
164+
);
165+
}
166+
167+
if ((action === ACTIONS.UPDATE && user_input.merge_tag) || (action === ACTIONS.ADD)) {
168+
const merge_tags = JSON.parse(user_input.merge_tag || '[]');
169+
data.tag = _.merge(
170+
data.tag,
171+
BucketSpaceFS._merge_reserved_tags(
172+
data.tag || BucketSpaceFS._default_bucket_tags(),
173+
merge_tags,
174+
action === ACTIONS.ADD ? true : await _is_bucket_empty(data),
175+
)
176+
);
177+
}
178+
157179
//if we're updating the owner, needs to override owner in file with the owner from user input.
158180
//if we're adding a bucket, need to set its owner id field
159181
if ((action === ACTIONS.UPDATE && user_input.owner) || (action === ACTIONS.ADD)) {
@@ -200,7 +222,24 @@ async function add_bucket(data) {
200222
data._id = mongo_utils.mongoObjectId();
201223
const parsed_bucket_data = await config_fs.create_bucket_config_file(data);
202224
await set_bucker_owner(parsed_bucket_data);
203-
return { code: ManageCLIResponse.BucketCreated, detail: parsed_bucket_data, event_arg: { bucket: data.name }};
225+
226+
const reserved_tag_event_args = data.tag?.reduce((curr, tag) => {
227+
const tag_info = config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[tag.key];
228+
229+
// If not a reserved tag - skip
230+
if (!tag_info) return curr;
231+
232+
// If no event is requested - skip
233+
if (!tag_info.event) return curr;
234+
235+
return Object.assign(curr, { [tag.key]: tag.value });
236+
}, {});
237+
238+
return {
239+
code: ManageCLIResponse.BucketCreated,
240+
detail: parsed_bucket_data,
241+
event_arg: { ...(reserved_tag_event_args || {}), bucket: data.name, account: parsed_bucket_data.bucket_owner },
242+
};
204243
}
205244

206245
/**
@@ -256,25 +295,14 @@ async function update_bucket(data) {
256295
*/
257296
async function delete_bucket(data, force) {
258297
try {
259-
const temp_dir_name = native_fs_utils.get_bucket_tmpdir_name(data._id);
298+
const bucket_empty = await _is_bucket_empty(data);
299+
if (!bucket_empty && !force) {
300+
throw_cli_error(ManageCLIError.BucketDeleteForbiddenHasObjects, data.name);
301+
}
302+
260303
const bucket_temp_dir_path = native_fs_utils.get_bucket_tmpdir_full_path(data.path, data._id);
261-
// fs_contexts for bucket temp dir (storage path)
262304
const fs_context_fs_backend = native_fs_utils.get_process_fs_context(data.fs_backend);
263-
let entries;
264-
try {
265-
entries = await nb_native().fs.readdir(fs_context_fs_backend, data.path);
266-
} catch (err) {
267-
dbg.warn(`delete_bucket: bucket name ${data.name},` +
268-
`got an error on readdir with path: ${data.path}`, err);
269-
// if the bucket's path was deleted first (encounter ENOENT error) - continue deletion
270-
if (err.code !== 'ENOENT') throw err;
271-
}
272-
if (entries) {
273-
const object_entries = entries.filter(element => !element.name.endsWith(temp_dir_name));
274-
if (object_entries.length > 0 && !force) {
275-
throw_cli_error(ManageCLIError.BucketDeleteForbiddenHasObjects, data.name);
276-
}
277-
}
305+
278306
await native_fs_utils.folder_delete(bucket_temp_dir_path, fs_context_fs_backend, true);
279307
await config_fs.delete_bucket_config_file(data.name);
280308
return { code: ManageCLIResponse.BucketDeleted, detail: { name: data.name }, event_arg: { bucket: data.name } };
@@ -340,6 +368,33 @@ async function list_bucket_config_files(wide, filters = {}) {
340368
return config_files_list;
341369
}
342370

371+
/**
372+
* _is_bucket_empty returns true if the given bucket is empty
373+
*
374+
* @param {*} data
375+
* @returns {Promise<boolean>}
376+
*/
377+
async function _is_bucket_empty(data) {
378+
const temp_dir_name = native_fs_utils.get_bucket_tmpdir_name(data._id);
379+
// fs_contexts for bucket temp dir (storage path)
380+
const fs_context_fs_backend = native_fs_utils.get_process_fs_context(data.fs_backend);
381+
let entries;
382+
try {
383+
entries = await nb_native().fs.readdir(fs_context_fs_backend, data.path);
384+
} catch (err) {
385+
dbg.warn(`_is_bucket_empty: bucket name ${data.name},` +
386+
`got an error on readdir with path: ${data.path}`, err);
387+
// if the bucket's path was deleted first (encounter ENOENT error) - continue deletion
388+
if (err.code !== 'ENOENT') throw err;
389+
}
390+
if (entries) {
391+
const object_entries = entries.filter(element => !element.name.endsWith(temp_dir_name));
392+
return object_entries.length === 0;
393+
}
394+
395+
return true;
396+
}
397+
343398
/**
344399
* bucket_management does the following -
345400
* 1. fetches the bucket data if this is not a list operation
@@ -361,7 +416,37 @@ async function bucket_management(action, user_input) {
361416
} else if (action === ACTIONS.STATUS) {
362417
response = await get_bucket_status(data);
363418
} else if (action === ACTIONS.UPDATE) {
364-
response = await update_bucket(data);
419+
const bucket_path = config_fs.get_bucket_path_by_name(user_input.name);
420+
const bucket_lock_file = `${bucket_path}.lock`;
421+
await native_fs_utils.lock_and_run(config_fs.fs_context, bucket_lock_file, async () => {
422+
const prev_bucket_info = await fetch_bucket_data(action, _.omit(user_input, ['tag', 'merge_tag']));
423+
const bucket_info = await fetch_bucket_data(action, user_input);
424+
425+
const tagging_object = prev_bucket_info.tag?.reduce((curr, tag) => Object.assign(curr, { [tag.key]: tag.value }), {});
426+
427+
let reserved_tag_modified = false;
428+
const reserved_tag_event_args = bucket_info.tag?.reduce((curr, tag) => {
429+
const tag_info = config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[tag.key];
430+
431+
// If not a reserved tag - skip
432+
if (!tag_info) return curr;
433+
434+
// If no event is requested - skip
435+
if (!tag_info.event) return curr;
436+
437+
// If value didn't change - skip
438+
if (_.isEqual(tagging_object[tag.key], tag.value)) return curr;
439+
440+
reserved_tag_modified = true;
441+
return Object.assign(curr, { [tag.key]: tag.value });
442+
}, {});
443+
444+
response = await update_bucket(bucket_info);
445+
if (reserved_tag_modified) {
446+
new NoobaaEvent(NoobaaEvent.BUCKET_RESERVED_TAG_MODIFIED)
447+
.create_event(undefined, { ...reserved_tag_event_args, bucket_name: user_input.name });
448+
}
449+
});
365450
} else if (action === ACTIONS.DELETE) {
366451
const force = get_boolean_or_string_value(user_input.force);
367452
response = await delete_bucket(data, force);
@@ -729,6 +814,7 @@ async function set_bucker_owner(bucket_data) {
729814
} catch (err) {
730815
dbg.warn(`set_bucker_owner.couldn't find bucket owner data by id ${bucket_data.owner_account}`);
731816
}
817+
console.log(account_data);
732818
bucket_data.bucket_owner = account_data?.name;
733819
}
734820

‎src/manage_nsfs/manage_nsfs_constants.js

+4-1
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ const VALID_OPTIONS_ANONYMOUS_ACCOUNT = {
6262

6363
const VALID_OPTIONS_BUCKET = {
6464
'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'force_md5_etag', 'notifications', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
65-
'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', ...CLI_MUTUAL_OPTIONS]),
65+
'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', 'tag', 'merge_tag', ...CLI_MUTUAL_OPTIONS]),
6666
'delete': new Set(['name', 'force', ...CLI_MUTUAL_OPTIONS]),
6767
'list': new Set(['wide', 'name', ...CLI_MUTUAL_OPTIONS]),
6868
'status': new Set(['name', ...CLI_MUTUAL_OPTIONS]),
@@ -171,6 +171,9 @@ const OPTION_TYPE = {
171171
key: 'string',
172172
value: 'string',
173173
remove_key: 'boolean',
174+
// bucket tagging
175+
tag: 'string',
176+
merge_tag: 'string',
174177
};
175178

176179
const BOOLEAN_STRING_VALUES = ['true', 'false'];

‎src/manage_nsfs/manage_nsfs_events_utils.js

+10
Original file line numberDiff line numberDiff line change
@@ -322,6 +322,16 @@ NoobaaEvent.UNAUTHORIZED = Object.freeze({
322322
severity: 'ERROR',
323323
state: 'HEALTHY',
324324
});
325+
NoobaaEvent.BUCKET_RESERVED_TAG_MODIFIED = Object.freeze({
326+
event_code: 'noobaa_bucket_reserved_tag_modified',
327+
message: 'Bucket reserved tag modified',
328+
description: 'Noobaa bucket reserved tag modified',
329+
entity_type: 'NODE',
330+
event_type: 'INFO',
331+
scope: 'NODE',
332+
severity: 'INFO',
333+
state: 'HEALTHY',
334+
});
325335

326336
NoobaaEvent.IO_STREAM_ITEM_TIMEOUT = Object.freeze({
327337
event_code: 'bucket_io_stream_item_timeout',

0 commit comments

Comments
 (0)