Skip to content

Commit 5cf067e

Browse files
tangledbytestangledbytes
committed
add support for reserved bucket tags
Signed-off-by: Utkarsh Srivastava <[email protected]> fix put_bucket_tagging test - pass dummy objectsdk Signed-off-by: Utkarsh Srivastava <[email protected]> add support for events, race safe tag manipulation and --merge_tag flag Signed-off-by: Utkarsh Srivastava <[email protected]> fix lint issues Signed-off-by: Utkarsh Srivastava <[email protected]> clarify the reserved tags config comment Signed-off-by: Utkarsh Srivastava <[email protected]>
1 parent 44d6518 commit 5cf067e

File tree

10 files changed

+250
-31
lines changed

10 files changed

+250
-31
lines changed

‎config.js

+27
Original file line numberDiff line numberDiff line change
@@ -956,6 +956,33 @@ config.NSFS_GLACIER_FORCE_EXPIRE_ON_GET = false;
956956
// interval
957957
config.NSFS_GLACIER_MIGRATE_LOG_THRESHOLD = 50 * 1024;
958958

959+
/**
960+
* NSFS_GLACIER_RESERVED_BUCKET_TAGS defines an object of bucket tags which will be reserved
961+
* by the system and PUT operations for them via S3 API would be limited - as in they would be
962+
* mutable only if specified and only under certain conditions.
963+
*
964+
* @type {Record<string, {
965+
* schema: Record<any, any> & { $id: string },
966+
* immutable: true | false | 'if-data',
967+
* default: any,
968+
* event: boolean
969+
* }>}
970+
*
971+
* @example
972+
* {
973+
'deep-archive-copies': {
974+
schema: {
975+
$id: 'deep-archive-copies-schema-v0',
976+
enum: ['1', '2']
977+
}, // JSON Schema
978+
immutable: 'if-data',
979+
default: '1',
980+
event: true
981+
}
982+
* }
983+
*/
984+
config.NSFS_GLACIER_RESERVED_BUCKET_TAGS = {};
985+
959986
// anonymous account name
960987
config.ANONYMOUS_ACCOUNT_NAME = 'anonymous';
961988

‎docs/NooBaaNonContainerized/Events.md

+5
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,11 @@ The following list includes events that indicate on a normal / successful operat
4343
- Arguments: `whitelist_ips`
4444
- Description: Whitelist Server IPs updated successfully using NooBaa CLI.
4545

46+
#### 7. `noobaa_bucket_reserved_tag_modified`
47+
- Arguments:
48+
- `bucket_name`
49+
- `<tag_value>` (if `event` is `true` for the reserved tag)
50+
- Description: NooBaa bucket reserved tag was modified successfully using NooBaa CLI or S3.
4651

4752
### Error Indicating Events
4853

‎src/cmd/manage_nsfs.js

+81-19
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ const { throw_cli_error, get_bucket_owner_account_by_name,
3939
const manage_nsfs_validations = require('../manage_nsfs/manage_nsfs_validations');
4040
const nc_mkm = require('../manage_nsfs/nc_master_key_manager').get_instance();
4141
const notifications_util = require('../util/notifications_util');
42+
const BucketSpaceFS = require('../sdk/bucketspace_fs');
43+
const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent;
4244

4345
///////////////
4446
//// GENERAL //
@@ -135,7 +137,6 @@ async function fetch_bucket_data(action, user_input) {
135137
force_md5_etag: user_input.force_md5_etag === undefined || user_input.force_md5_etag === '' ? user_input.force_md5_etag : get_boolean_or_string_value(user_input.force_md5_etag),
136138
notifications: user_input.notifications
137139
};
138-
139140
if (user_input.bucket_policy !== undefined) {
140141
if (typeof user_input.bucket_policy === 'string') {
141142
// bucket_policy deletion specified with empty string ''
@@ -154,6 +155,27 @@ async function fetch_bucket_data(action, user_input) {
154155
data = await merge_new_and_existing_config_data(data);
155156
}
156157

158+
if ((action === ACTIONS.UPDATE && user_input.tag) || (action === ACTIONS.ADD)) {
159+
const tags = JSON.parse(user_input.tag || '[]');
160+
data.tag = BucketSpaceFS._merge_reserved_tags(
161+
data.tag || BucketSpaceFS._default_bucket_tags(),
162+
tags,
163+
action === ACTIONS.ADD ? true : await _is_bucket_empty(data),
164+
);
165+
}
166+
167+
if ((action === ACTIONS.UPDATE && user_input.merge_tag) || (action === ACTIONS.ADD)) {
168+
const merge_tags = JSON.parse(user_input.merge_tag || '[]');
169+
data.tag = _.merge(
170+
data.tag,
171+
BucketSpaceFS._merge_reserved_tags(
172+
data.tag || BucketSpaceFS._default_bucket_tags(),
173+
merge_tags,
174+
action === ACTIONS.ADD ? true : await _is_bucket_empty(data),
175+
)
176+
);
177+
}
178+
157179
//if we're updating the owner, needs to override owner in file with the owner from user input.
158180
//if we're adding a bucket, need to set its owner id field
159181
if ((action === ACTIONS.UPDATE && user_input.owner) || (action === ACTIONS.ADD)) {
@@ -256,25 +278,14 @@ async function update_bucket(data) {
256278
*/
257279
async function delete_bucket(data, force) {
258280
try {
259-
const temp_dir_name = native_fs_utils.get_bucket_tmpdir_name(data._id);
281+
const bucket_empty = await _is_bucket_empty(data);
282+
if (!bucket_empty && !force) {
283+
throw_cli_error(ManageCLIError.BucketDeleteForbiddenHasObjects, data.name);
284+
}
285+
260286
const bucket_temp_dir_path = native_fs_utils.get_bucket_tmpdir_full_path(data.path, data._id);
261-
// fs_contexts for bucket temp dir (storage path)
262287
const fs_context_fs_backend = native_fs_utils.get_process_fs_context(data.fs_backend);
263-
let entries;
264-
try {
265-
entries = await nb_native().fs.readdir(fs_context_fs_backend, data.path);
266-
} catch (err) {
267-
dbg.warn(`delete_bucket: bucket name ${data.name},` +
268-
`got an error on readdir with path: ${data.path}`, err);
269-
// if the bucket's path was deleted first (encounter ENOENT error) - continue deletion
270-
if (err.code !== 'ENOENT') throw err;
271-
}
272-
if (entries) {
273-
const object_entries = entries.filter(element => !element.name.endsWith(temp_dir_name));
274-
if (object_entries.length > 0 && !force) {
275-
throw_cli_error(ManageCLIError.BucketDeleteForbiddenHasObjects, data.name);
276-
}
277-
}
288+
278289
await native_fs_utils.folder_delete(bucket_temp_dir_path, fs_context_fs_backend, true);
279290
await config_fs.delete_bucket_config_file(data.name);
280291
return { code: ManageCLIResponse.BucketDeleted, detail: { name: data.name }, event_arg: { bucket: data.name } };
@@ -340,6 +351,27 @@ async function list_bucket_config_files(wide, filters = {}) {
340351
return config_files_list;
341352
}
342353

354+
async function _is_bucket_empty(data) {
355+
const temp_dir_name = native_fs_utils.get_bucket_tmpdir_name(data._id);
356+
// fs_contexts for bucket temp dir (storage path)
357+
const fs_context_fs_backend = native_fs_utils.get_process_fs_context(data.fs_backend);
358+
let entries;
359+
try {
360+
entries = await nb_native().fs.readdir(fs_context_fs_backend, data.path);
361+
} catch (err) {
362+
dbg.warn(`_is_bucket_empty: bucket name ${data.name},` +
363+
`got an error on readdir with path: ${data.path}`, err);
364+
// if the bucket's path was deleted first (encounter ENOENT error) - continue deletion
365+
if (err.code !== 'ENOENT') throw err;
366+
}
367+
if (entries) {
368+
const object_entries = entries.filter(element => !element.name.endsWith(temp_dir_name));
369+
return object_entries.length === 0;
370+
}
371+
372+
return true;
373+
}
374+
343375
/**
344376
* bucket_management does the following -
345377
* 1. fetches the bucket data if this is not a list operation
@@ -361,7 +393,37 @@ async function bucket_management(action, user_input) {
361393
} else if (action === ACTIONS.STATUS) {
362394
response = await get_bucket_status(data);
363395
} else if (action === ACTIONS.UPDATE) {
364-
response = await update_bucket(data);
396+
const bucket_path = config_fs.get_bucket_path_by_name(user_input.name);
397+
const bucket_lock_file = `${bucket_path}.lock`;
398+
await native_fs_utils.lock_and_run(config_fs.fs_context, bucket_lock_file, async () => {
399+
const prev_bucket_info = await fetch_bucket_data(action, _.omit(user_input, ['tag', 'merge_tag']));
400+
const bucket_info = await fetch_bucket_data(action, user_input);
401+
402+
const tagging_object = prev_bucket_info.tag?.reduce((curr, tag) => Object.assign(curr, { [tag.key]: tag.value }), {});
403+
404+
let reserved_tag_modified = false;
405+
const reserved_tag_event_args = bucket_info.tag?.reduce((curr, tag) => {
406+
const tag_info = config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[tag.key];
407+
408+
// If not a reserved tag - skip
409+
if (!tag_info) return curr;
410+
411+
// If no event is requested - skip
412+
if (!tag_info.event) return curr;
413+
414+
// If value didn't change - skip
415+
if (_.isEqual(tagging_object[tag.key], tag.value)) return curr;
416+
417+
reserved_tag_modified = true;
418+
return Object.assign(curr, { [tag.key]: tag.value });
419+
}, {});
420+
421+
response = await update_bucket(bucket_info);
422+
if (reserved_tag_modified) {
423+
new NoobaaEvent(NoobaaEvent.BUCKET_RESERVED_TAG_MODIFIED)
424+
.create_event(undefined, { ...reserved_tag_event_args, bucket_name: user_input.name });
425+
}
426+
});
365427
} else if (action === ACTIONS.DELETE) {
366428
const force = get_boolean_or_string_value(user_input.force);
367429
response = await delete_bucket(data, force);

‎src/manage_nsfs/manage_nsfs_constants.js

+4-1
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ const VALID_OPTIONS_ANONYMOUS_ACCOUNT = {
6262

6363
const VALID_OPTIONS_BUCKET = {
6464
'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'force_md5_etag', 'notifications', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
65-
'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', ...CLI_MUTUAL_OPTIONS]),
65+
'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', 'tag', 'merge_tag', ...CLI_MUTUAL_OPTIONS]),
6666
'delete': new Set(['name', 'force', ...CLI_MUTUAL_OPTIONS]),
6767
'list': new Set(['wide', 'name', ...CLI_MUTUAL_OPTIONS]),
6868
'status': new Set(['name', ...CLI_MUTUAL_OPTIONS]),
@@ -171,6 +171,9 @@ const OPTION_TYPE = {
171171
key: 'string',
172172
value: 'string',
173173
remove_key: 'boolean',
174+
// bucket tagging
175+
tag: 'string',
176+
merge_tag: 'string',
174177
};
175178

176179
const BOOLEAN_STRING_VALUES = ['true', 'false'];

‎src/manage_nsfs/manage_nsfs_events_utils.js

+10
Original file line numberDiff line numberDiff line change
@@ -322,6 +322,16 @@ NoobaaEvent.UNAUTHORIZED = Object.freeze({
322322
severity: 'ERROR',
323323
state: 'HEALTHY',
324324
});
325+
NoobaaEvent.BUCKET_RESERVED_TAG_MODIFIED = Object.freeze({
326+
event_code: 'noobaa_bucket_reserved_tag_modified',
327+
message: 'Bucket reserved tag modified',
328+
description: 'Noobaa bucket reserved tag modified',
329+
entity_type: 'NODE',
330+
event_type: 'INFO',
331+
scope: 'NODE',
332+
severity: 'INFO',
333+
state: 'HEALTHY',
334+
});
325335

326336
NoobaaEvent.IO_STREAM_ITEM_TIMEOUT = Object.freeze({
327337
event_code: 'bucket_io_stream_item_timeout',

0 commit comments

Comments
 (0)