diff --git a/0019-cmake-modules-CheckCxxAtomic.cmake.patch b/0019-cmake-modules-CheckCxxAtomic.cmake.patch index b8c0186d3bfb1a62b5f55106a087edbcfebc0dd2..f353e3c370f04055e98d65a301b0d59842ba1b5e 100644 --- a/0019-cmake-modules-CheckCxxAtomic.cmake.patch +++ b/0019-cmake-modules-CheckCxxAtomic.cmake.patch @@ -1,10 +1,43 @@ ---- ceph-17.1.0/cmake/modules/CheckCxxAtomic.cmake.orig 2022-03-07 15:01:49.809505648 -0500 -+++ ceph-17.1.0/cmake/modules/CheckCxxAtomic.cmake 2022-03-09 12:47:31.018446185 -0500 -@@ -21,6 +21,7 @@ +--- ceph-17.2.0-359-gb2fe9ec8/cmake/modules/CheckCxxAtomic.cmake.orig 2022-06-03 08:45:32.341075140 -0400 ++++ ceph-17.2.0-359-gb2fe9ec8/cmake/modules/CheckCxxAtomic.cmake 2022-06-03 08:46:47.195775813 -0400 +@@ -10,8 +10,9 @@ + check_cxx_source_compiles(" + #include + #include ++#include + +-#if defined(__s390x__) || defined(__mips__) ++#if defined(__SIZEOF_INT128__) + // Boost needs 16-byte atomics for tagged pointers. + // These are implemented via inline instructions on the platform + // if 16-byte alignment can be proven, and are delegated to libatomic +@@ -21,13 +22,27 @@ // We specifically test access via an otherwise unknown pointer here // to ensure we get the most complex case. If this access can be // done without libatomic, then all accesses can be done. -+bool atomic16(std::atomic *ptr) __attribute__ ((used)); - bool atomic16(std::atomic *ptr) +-bool atomic16(std::atomic *ptr) ++struct tagged_ptr { ++ int* ptr; ++ std::size_t tag; ++}; ++ ++void atomic16(std::atomic *ptr) __attribute__ ((used)); ++void atomic16(std::atomic *ptr) { - return *ptr != 0; +- return *ptr != 0; ++ tagged_ptr p{nullptr, 1}; ++ ptr->store(p); ++ tagged_ptr f = ptr->load(); ++ tagged_ptr new_tag{nullptr, 0}; ++ ptr->compare_exchange_strong(f, new_tag); + } + #endif + + int main() { ++#if defined(__SIZEOF_INT128__) ++ std::atomic ptr; ++ atomic16(&ptr); ++#endif + std::atomic w1; + std::atomic w2; + std::atomic w4; diff --git a/0021-cephfs-shell.patch b/0021-cephfs-shell.patch new file mode 100644 index 0000000000000000000000000000000000000000..f85a91efc5caf1afeab1be41f13111f7aa884716 --- /dev/null +++ b/0021-cephfs-shell.patch @@ -0,0 +1,1756 @@ +--- ceph-17.2.1/src/tools/cephfs/CMakeLists.txt.orig 2022-07-05 19:26:04.629170597 -0400 ++++ ceph-17.2.1/src/tools/cephfs/CMakeLists.txt 2022-07-05 19:26:40.710580427 -0400 +@@ -49,12 +49,7 @@ + + option(WITH_CEPHFS_SHELL "install cephfs-shell" OFF) + if(WITH_CEPHFS_SHELL) +- include(Distutils) +- distutils_install_module(cephfs-shell) +- if(WITH_TESTS) +- include(AddCephTest) +- add_tox_test(cephfs-shell) +- endif() ++ add_subdirectory(shell) + endif() + + option(WITH_CEPHFS_TOP "install cephfs-top utility" ON) +--- /dev/null 2022-06-30 09:45:32.996000000 -0400 ++++ ceph-17.2.1/src/tools/cephfs/shell/CMakeLists.txt 2022-07-05 19:27:58.983300150 -0400 +@@ -0,0 +1,7 @@ ++include(Distutils) ++distutils_install_module(cephfs-shell) ++ ++if(WITH_TESTS) ++ include(AddCephTest) ++ add_tox_test(cephfs-shell) ++endif() +--- /dev/null 2022-06-30 09:45:32.996000000 -0400 ++++ ceph-17.2.1/src/tools/cephfs/shell/cephfs-shell 2022-06-23 10:41:35.000000000 -0400 +@@ -0,0 +1,1687 @@ ++#!/usr/bin/python3 ++# coding = utf-8 ++ ++import argparse ++import os ++import os.path ++import sys ++import cephfs as libcephfs ++import shutil ++import traceback ++import colorama ++import fnmatch ++import math ++import re ++import shlex ++import stat ++import errno ++ ++from cmd2 import Cmd ++from cmd2 import __version__ as cmd2_version ++from distutils.version import LooseVersion ++ ++if sys.version_info.major < 3: ++ raise RuntimeError("cephfs-shell is only compatible with python3") ++ ++try: ++ from cmd2 import with_argparser ++except ImportError: ++ def with_argparser(argparser): ++ import functools ++ ++ def argparser_decorator(func): ++ @functools.wraps(func) ++ def wrapper(thiz, cmdline): ++ if isinstance(cmdline, list): ++ arglist = cmdline ++ else: ++ # do not split if it's already a list ++ arglist = shlex.split(cmdline, posix=False) ++ # in case user quotes the command args ++ arglist = [arg.strip('\'""') for arg in arglist] ++ try: ++ args = argparser.parse_args(arglist) ++ except SystemExit: ++ shell.exit_code = 1 ++ # argparse exits at seeing bad arguments ++ return ++ else: ++ return func(thiz, args) ++ argparser.prog = func.__name__[3:] ++ if argparser.description is None and func.__doc__: ++ argparser.description = func.__doc__ ++ ++ return wrapper ++ ++ return argparser_decorator ++ ++ ++cephfs = None # holds CephFS Python bindings ++shell = None # holds instance of class CephFSShell ++exit_codes = {'Misc': 1, ++ 'KeyboardInterrupt': 2, ++ errno.EPERM: 3, ++ errno.EACCES: 4, ++ errno.ENOENT: 5, ++ errno.EIO: 6, ++ errno.ENOSPC: 7, ++ errno.EEXIST: 8, ++ errno.ENODATA: 9, ++ errno.EINVAL: 10, ++ errno.EOPNOTSUPP: 11, ++ errno.ERANGE: 12, ++ errno.EWOULDBLOCK: 13, ++ errno.ENOTEMPTY: 14, ++ errno.ENOTDIR: 15, ++ errno.EDQUOT: 16, ++ errno.EPIPE: 17, ++ errno.ESHUTDOWN: 18, ++ errno.ECONNABORTED: 19, ++ errno.ECONNREFUSED: 20, ++ errno.ECONNRESET: 21, ++ errno.EINTR: 22} ++ ++ ++######################################################################### ++# ++# Following are methods are generically useful through class CephFSShell ++# ++####################################################################### ++ ++ ++def poutput(s, end='\n'): ++ shell.poutput(s, end=end) ++ ++ ++def perror(msg, **kwargs): ++ shell.perror(msg, **kwargs) ++ ++ ++def set_exit_code_msg(errcode='Misc', msg=''): ++ """ ++ Set exit code and print error message ++ """ ++ if isinstance(msg, libcephfs.Error): ++ shell.exit_code = exit_codes[msg.get_error_code()] ++ else: ++ shell.exit_code = exit_codes[errcode] ++ if msg: ++ perror(msg) ++ ++ ++def mode_notation(mode): ++ """ ++ """ ++ permission_bits = {'0': '---', ++ '1': '--x', ++ '2': '-w-', ++ '3': '-wx', ++ '4': 'r--', ++ '5': 'r-x', ++ '6': 'rw-', ++ '7': 'rwx'} ++ mode = str(oct(mode)) ++ notation = '-' ++ if mode[2] == '4': ++ notation = 'd' ++ elif mode[2:4] == '12': ++ notation = 'l' ++ for i in mode[-3:]: ++ notation += permission_bits[i] ++ return notation ++ ++ ++def get_chunks(file_size): ++ chunk_start = 0 ++ chunk_size = 0x20000 # 131072 bytes, default max ssl buffer size ++ while chunk_start + chunk_size < file_size: ++ yield(chunk_start, chunk_size) ++ chunk_start += chunk_size ++ final_chunk_size = file_size - chunk_start ++ yield(chunk_start, final_chunk_size) ++ ++ ++def to_bytes(param): ++ # don't convert as follows as it can lead unusable results like coverting ++ # [1, 2, 3, 4] to '[1, 2, 3, 4]' - ++ # str(param).encode('utf-8') ++ if isinstance(param, bytes): ++ return param ++ elif isinstance(param, str): ++ return bytes(param, encoding='utf-8') ++ elif isinstance(param, list): ++ return [i.encode('utf-8') if isinstance(i, str) else to_bytes(i) for ++ i in param] ++ elif isinstance(param, int) or isinstance(param, float): ++ return str(param).encode('utf-8') ++ elif param is None: ++ return None ++ ++ ++def ls(path, opts=''): ++ # opts tries to be like /bin/ls opts ++ almost_all = 'A' in opts ++ try: ++ with cephfs.opendir(path) as d: ++ while True: ++ dent = cephfs.readdir(d) ++ if dent is None: ++ return ++ elif almost_all and dent.d_name in (b'.', b'..'): ++ continue ++ yield dent ++ except libcephfs.ObjectNotFound as e: ++ set_exit_code_msg(msg=e) ++ ++ ++def glob(path, pattern): ++ paths = [] ++ parent_dir = os.path.dirname(path) ++ if parent_dir == b'': ++ parent_dir = b'/' ++ if path == b'/' or is_dir_exists(os.path.basename(path), parent_dir): ++ for i in ls(path, opts='A'): ++ if fnmatch.fnmatch(i.d_name, pattern): ++ paths.append(os.path.join(path, i.d_name)) ++ return paths ++ ++ ++def locate_file(name, case_sensitive=True): ++ dir_list = sorted(set(dirwalk(cephfs.getcwd()))) ++ if not case_sensitive: ++ return [dname for dname in dir_list if name.lower() in dname.lower()] ++ else: ++ return [dname for dname in dir_list if name in dname] ++ ++ ++def get_all_possible_paths(pattern): ++ complete_pattern = pattern[:] ++ paths = [] ++ is_rel_path = not os.path.isabs(pattern) ++ if is_rel_path: ++ dir_ = cephfs.getcwd() ++ else: ++ dir_ = b'/' ++ pattern = pattern[1:] ++ patterns = pattern.split(b'/') ++ paths.extend(glob(dir_, patterns[0])) ++ patterns.pop(0) ++ for pattern in patterns: ++ for path in paths: ++ paths.extend(glob(path, pattern)) ++ if is_rel_path: ++ complete_pattern = os.path.join(cephfs.getcwd(), complete_pattern) ++ return [path for path in paths if fnmatch.fnmatch(path, complete_pattern)] ++ ++ ++suffixes = ['B', 'K', 'M', 'G', 'T', 'P'] ++ ++ ++def humansize(nbytes): ++ i = 0 ++ while nbytes >= 1024 and i < len(suffixes) - 1: ++ nbytes /= 1024. ++ i += 1 ++ nbytes = math.ceil(nbytes) ++ f = ('%d' % nbytes).rstrip('.') ++ return '%s%s' % (f, suffixes[i]) ++ ++ ++def style_listing(path, is_dir, is_symlink, ls_long=False): ++ if not (is_dir or is_symlink): ++ return path ++ pretty = colorama.Style.BRIGHT ++ if is_symlink: ++ pretty += colorama.Fore.CYAN + path ++ if ls_long: ++ # Add target path ++ pretty += ' -> ' + cephfs.readlink(path, size=255).decode('utf-8') ++ elif is_dir: ++ pretty += colorama.Fore.BLUE + path + '/' ++ pretty += colorama.Style.RESET_ALL ++ return pretty ++ ++ ++def print_long(path, is_dir, is_symlink, human_readable): ++ info = cephfs.stat(path, follow_symlink=(not is_symlink)) ++ pretty = style_listing(os.path.basename(path.decode('utf-8')), is_dir, is_symlink, True) ++ if human_readable: ++ sizefmt = '\t {:10s}'.format(humansize(info.st_size)) ++ else: ++ sizefmt = '{:12d}'.format(info.st_size) ++ poutput(f'{mode_notation(info.st_mode)} {sizefmt} {info.st_uid} {info.st_gid} {info.st_mtime}' ++ f' {pretty}') ++ ++ ++def word_len(word): ++ """ ++ Returns the word length, minus any color codes. ++ """ ++ if word[0] == '\x1b': ++ return len(word) - 9 ++ return len(word) ++ ++ ++def is_dir_exists(path, dir_=b''): ++ path_to_stat = os.path.join(dir_, path) ++ try: ++ return ((cephfs.stat(path_to_stat).st_mode & 0o0040000) != 0) ++ except libcephfs.Error: ++ return False ++ ++ ++def is_file_exists(path, dir_=b''): ++ try: ++ # if its not a directory, then its a file ++ return ((cephfs.stat(os.path.join(dir_, path)).st_mode & 0o0040000) == 0) ++ except libcephfs.Error: ++ return False ++ ++ ++def print_list(words, termwidth=79): ++ if not words: ++ return ++ words = [word.decode('utf-8') if isinstance(word, bytes) else word for word in words] ++ width = max([word_len(word) for word in words]) + 2 ++ nwords = len(words) ++ ncols = max(1, (termwidth + 1) // (width + 1)) ++ nrows = (nwords + ncols - 1) // ncols ++ for row in range(nrows): ++ for i in range(row, nwords, nrows): ++ word = words[i] ++ print_width = width ++ if word[0] == '\x1b': ++ print_width = print_width + 10 ++ ++ poutput('%-*s' % (print_width, words[i]), ++ end='\n' if i + nrows >= nwords else '') ++ ++ ++def copy_from_local(local_path, remote_path): ++ stdin = -1 ++ file_ = None ++ fd = None ++ convert_to_bytes = False ++ if local_path == b'-': ++ file_ = sys.stdin ++ convert_to_bytes = True ++ else: ++ try: ++ file_ = open(local_path, 'rb') ++ except PermissionError as e: ++ set_exit_code_msg(e.errno, 'error: no permission to read local file {}'.format( ++ local_path.decode('utf-8'))) ++ return ++ stdin = 1 ++ try: ++ fd = cephfs.open(remote_path, 'w', 0o666) ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ return ++ progress = 0 ++ while True: ++ data = file_.read(65536) ++ if not data or len(data) == 0: ++ break ++ if convert_to_bytes: ++ data = to_bytes(data) ++ wrote = cephfs.write(fd, data, progress) ++ if wrote < 0: ++ break ++ progress += wrote ++ cephfs.close(fd) ++ if stdin > 0: ++ file_.close() ++ poutput('') ++ ++ ++def copy_to_local(remote_path, local_path): ++ fd = None ++ if local_path != b'-': ++ local_dir = os.path.dirname(local_path) ++ dir_list = remote_path.rsplit(b'/', 1) ++ if not os.path.exists(local_dir): ++ os.makedirs(local_dir) ++ if len(dir_list) > 2 and dir_list[1] == b'': ++ return ++ fd = open(local_path, 'wb+') ++ file_ = cephfs.open(remote_path, 'r') ++ file_size = cephfs.stat(remote_path).st_size ++ if file_size <= 0: ++ return ++ progress = 0 ++ for chunk_start, chunk_size in get_chunks(file_size): ++ file_chunk = cephfs.read(file_, chunk_start, chunk_size) ++ progress += len(file_chunk) ++ if fd: ++ fd.write(file_chunk) ++ else: ++ poutput(file_chunk.decode('utf-8')) ++ cephfs.close(file_) ++ if fd: ++ fd.close() ++ ++ ++def dirwalk(path): ++ """ ++ walk a directory tree, using a generator ++ """ ++ path = os.path.normpath(path) ++ for item in ls(path, opts='A'): ++ fullpath = os.path.join(path, item.d_name) ++ src_path = fullpath.rsplit(b'/', 1)[0] ++ ++ yield os.path.normpath(fullpath) ++ if is_dir_exists(item.d_name, src_path): ++ for x in dirwalk(fullpath): ++ yield x ++ ++ ++################################################################## ++# ++# Following methods are implementation for CephFS Shell commands ++# ++################################################################# ++ ++class CephFSShell(Cmd): ++ ++ def __init__(self): ++ super().__init__() ++ self.working_dir = cephfs.getcwd().decode('utf-8') ++ self.set_prompt() ++ self.interactive = False ++ self.umask = '2' ++ ++ def default(self, line): ++ perror('Unrecognized command') ++ ++ def set_prompt(self): ++ self.prompt = ('\033[01;33mCephFS:~' + colorama.Fore.LIGHTCYAN_EX ++ + self.working_dir + colorama.Style.RESET_ALL ++ + '\033[01;33m>>>\033[00m ') ++ ++ def create_argparser(self, command): ++ try: ++ argparse_args = getattr(self, 'argparse_' + command) ++ except AttributeError: ++ set_exit_code_msg() ++ return None ++ doc_lines = getattr( ++ self, 'do_' + command).__doc__.expandtabs().splitlines() ++ if '' in doc_lines: ++ blank_idx = doc_lines.index('') ++ usage = doc_lines[:blank_idx] ++ description = doc_lines[blank_idx + 1:] ++ else: ++ usage = doc_lines ++ description = [] ++ parser = argparse.ArgumentParser( ++ prog=command, ++ usage='\n'.join(usage), ++ description='\n'.join(description), ++ formatter_class=argparse.ArgumentDefaultsHelpFormatter ++ ) ++ for args, kwargs in argparse_args: ++ parser.add_argument(*args, **kwargs) ++ return parser ++ ++ def complete_filenames(self, text, line, begidx, endidx): ++ if not text: ++ completions = [x.d_name.decode('utf-8') + '/' * int(x.is_dir()) ++ for x in ls(b".", opts='A')] ++ else: ++ if text.count('/') > 0: ++ completions = [text.rsplit('/', 1)[0] + '/' ++ + x.d_name.decode('utf-8') + '/' ++ * int(x.is_dir()) for x in ls('/' ++ + text.rsplit('/', 1)[0], opts='A') ++ if x.d_name.decode('utf-8').startswith( ++ text.rsplit('/', 1)[1])] ++ else: ++ completions = [x.d_name.decode('utf-8') + '/' ++ * int(x.is_dir()) for x in ls(b".", opts='A') ++ if x.d_name.decode('utf-8').startswith(text)] ++ if len(completions) == 1 and completions[0][-1] == '/': ++ dir_, file_ = completions[0].rsplit('/', 1) ++ completions.extend([dir_ + '/' + x.d_name.decode('utf-8') ++ + '/' * int(x.is_dir()) for x in ++ ls('/' + dir_, opts='A') ++ if x.d_name.decode('utf-8').startswith(file_)]) ++ return self.delimiter_complete(text, line, begidx, endidx, completions, '/') ++ return completions ++ ++ def onecmd(self, line, **kwargs): ++ """ ++ Global error catcher ++ """ ++ try: ++ res = Cmd.onecmd(self, line, **kwargs) ++ if self.interactive: ++ self.set_prompt() ++ return res ++ except ConnectionError as e: ++ set_exit_code_msg(e.errno, f'***\n{e}') ++ except KeyboardInterrupt: ++ set_exit_code_msg('KeyboardInterrupt', 'Command aborted') ++ except (libcephfs.Error, Exception) as e: ++ if shell.debug: ++ traceback.print_exc(file=sys.stdout) ++ set_exit_code_msg(msg=e) ++ ++ class path_to_bytes(argparse.Action): ++ def __call__(self, parser, namespace, values, option_string=None): ++ values = to_bytes(values) ++ setattr(namespace, self.dest, values) ++ ++ # TODO: move the necessary contents from here to `class path_to_bytes`. ++ class get_list_of_bytes_path(argparse.Action): ++ def __call__(self, parser, namespace, values, option_string=None): ++ values = to_bytes(values) ++ ++ if values == b'.': ++ values = cephfs.getcwd() ++ else: ++ for i in values: ++ if i == b'.': ++ values[values.index(i)] = cephfs.getcwd() ++ ++ setattr(namespace, self.dest, values) ++ ++ def complete_mkdir(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ class ModeAction(argparse.Action): ++ def __init__(self, option_strings, dest, nargs=None, **kwargs): ++ if nargs is not None and nargs != '?': ++ raise ValueError("more than one modes not allowed") ++ super().__init__(option_strings, dest, **kwargs) ++ ++ def __call__(self, parser, namespace, values, option_string=None): ++ o_mode = 0 ++ res = None ++ try: ++ o_mode = int(values, base=8) ++ except ValueError: ++ res = re.match('((u?g?o?)|(a?))(=)(r?w?x?)', values) ++ if res is None: ++ parser.error("invalid mode: %s\n" ++ "mode must be a numeric octal literal\n" ++ "or ((u?g?o?)|(a?))(=)(r?w?x?)" % ++ values) ++ else: ++ # we are supporting only assignment of mode and not + or - ++ # as is generally available with the chmod command ++ # eg. ++ # >>> res = re.match('((u?g?o?)|(a?))(=)(r?w?x?)', 'go=') ++ # >>> res.groups() ++ # ('go', 'go', None, '=', '') ++ val = res.groups() ++ ++ if val[3] != '=': ++ parser.error("need assignment operator between user " ++ "and mode specifiers") ++ if val[4] == '': ++ parser.error("invalid mode: %s\n" ++ "mode must be combination of: r | w | x" % ++ values) ++ users = '' ++ if val[2] is None: ++ users = val[1] ++ else: ++ users = val[2] ++ ++ t_mode = 0 ++ if users == 'a': ++ users = 'ugo' ++ ++ if 'r' in val[4]: ++ t_mode |= 4 ++ if 'w' in val[4]: ++ t_mode |= 2 ++ if 'x' in val[4]: ++ t_mode |= 1 ++ ++ if 'u' in users: ++ o_mode |= (t_mode << 6) ++ if 'g' in users: ++ o_mode |= (t_mode << 3) ++ if 'o' in users: ++ o_mode |= t_mode ++ ++ if o_mode < 0: ++ parser.error("invalid mode: %s\n" ++ "mode cannot be negative" % values) ++ if o_mode > 0o777: ++ parser.error("invalid mode: %s\n" ++ "mode cannot be greater than octal 0777" % values) ++ ++ setattr(namespace, self.dest, str(oct(o_mode))) ++ ++ mkdir_parser = argparse.ArgumentParser( ++ description='Create the directory(ies), if they do not already exist.') ++ mkdir_parser.add_argument('dirs', type=str, ++ action=path_to_bytes, ++ metavar='DIR_NAME', ++ help='Name of new_directory.', ++ nargs='+') ++ mkdir_parser.add_argument('-m', '--mode', type=str, ++ action=ModeAction, ++ help='Sets the access mode for the new directory.') ++ mkdir_parser.add_argument('-p', '--parent', action='store_true', ++ help='Create parent directories as necessary. ' ++ 'When this option is specified, no error is' ++ 'reported if a directory already exists.') ++ ++ @with_argparser(mkdir_parser) ++ def do_mkdir(self, args): ++ """ ++ Create directory. ++ """ ++ for path in args.dirs: ++ if args.mode: ++ permission = int(args.mode, 8) ++ else: ++ permission = 0o777 ++ if args.parent: ++ cephfs.mkdirs(path, permission) ++ else: ++ try: ++ cephfs.mkdir(path, permission) ++ except libcephfs.Error as e: ++ set_exit_code_msg(e) ++ ++ def complete_put(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ index_dict = {1: self.path_complete} ++ return self.index_based_complete(text, line, begidx, endidx, index_dict) ++ ++ put_parser = argparse.ArgumentParser( ++ description='Copy a file/directory to Ceph File System from Local File System.') ++ put_parser.add_argument('local_path', type=str, action=path_to_bytes, ++ help='Path of the file in the local system') ++ put_parser.add_argument('remote_path', type=str, action=path_to_bytes, ++ help='Path of the file in the remote system') ++ put_parser.add_argument('-f', '--force', action='store_true', ++ help='Overwrites the destination if it already exists.') ++ ++ @with_argparser(put_parser) ++ def do_put(self, args): ++ """ ++ Copy a local file/directory to CephFS. ++ """ ++ if args.local_path != b'-' and not os.path.isfile(args.local_path) \ ++ and not os.path.isdir(args.local_path): ++ set_exit_code_msg(errno.ENOENT, ++ msg=f"error: " ++ f"{args.local_path.decode('utf-8')}: " ++ f"No such file or directory") ++ return ++ ++ if (is_file_exists(args.remote_path) or is_dir_exists( ++ args.remote_path)) and not args.force: ++ set_exit_code_msg(msg=f"error: file/directory " ++ f"{args.remote_path.decode('utf-8')} " ++ f"exists, use --force to overwrite") ++ return ++ ++ root_src_dir = args.local_path ++ root_dst_dir = args.remote_path ++ if args.local_path == b'.' or args.local_path == b'./': ++ root_src_dir = os.getcwdb() ++ elif len(args.local_path.rsplit(b'/', 1)) < 2: ++ root_src_dir = os.path.join(os.getcwdb(), args.local_path) ++ else: ++ p = args.local_path.split(b'/') ++ if p[0] == b'.': ++ root_src_dir = os.getcwdb() ++ p.pop(0) ++ while len(p) > 0: ++ root_src_dir += b'/' + p.pop(0) ++ ++ if root_dst_dir == b'.': ++ if args.local_path != b'-': ++ root_dst_dir = root_src_dir.rsplit(b'/', 1)[1] ++ if root_dst_dir == b'': ++ root_dst_dir = root_src_dir.rsplit(b'/', 1)[0] ++ a = root_dst_dir.rsplit(b'/', 1) ++ if len(a) > 1: ++ root_dst_dir = a[1] ++ else: ++ root_dst_dir = a[0] ++ else: ++ set_exit_code_msg(errno.EINVAL, 'error: no filename specified ' ++ 'for destination') ++ return ++ ++ if root_dst_dir[-1] != b'/': ++ root_dst_dir += b'/' ++ ++ if args.local_path == b'-' or os.path.isfile(root_src_dir): ++ if args.local_path == b'-': ++ root_src_dir = b'-' ++ copy_from_local(root_src_dir, root_dst_dir) ++ else: ++ for src_dir, dirs, files in os.walk(root_src_dir): ++ if isinstance(src_dir, str): ++ src_dir = to_bytes(src_dir) ++ dst_dir = src_dir.replace(root_src_dir, root_dst_dir, 1) ++ dst_dir = re.sub(rb'\/+', b'/', cephfs.getcwd() ++ + dst_dir) ++ if args.force and dst_dir != b'/' and not is_dir_exists( ++ dst_dir[:-1]) and not locate_file(dst_dir): ++ try: ++ cephfs.mkdirs(dst_dir, 0o777) ++ except libcephfs.Error: ++ pass ++ if (not args.force) and dst_dir != b'/' and not is_dir_exists( ++ dst_dir) and not os.path.isfile(root_src_dir): ++ try: ++ cephfs.mkdirs(dst_dir, 0o777) ++ except libcephfs.Error: ++ # TODO: perhaps, set retval to 1? ++ pass ++ ++ for dir_ in dirs: ++ dir_name = os.path.join(dst_dir, dir_) ++ if not is_dir_exists(dir_name): ++ try: ++ cephfs.mkdirs(dir_name, 0o777) ++ except libcephfs.Error: ++ # TODO: perhaps, set retval to 1? ++ pass ++ ++ for file_ in files: ++ src_file = os.path.join(src_dir, file_) ++ dst_file = re.sub(rb'\/+', b'/', b'/' + dst_dir + b'/' + file_) ++ if (not args.force) and is_file_exists(dst_file): ++ return ++ copy_from_local(src_file, os.path.join(cephfs.getcwd(), ++ dst_file)) ++ ++ def complete_get(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ get_parser = argparse.ArgumentParser( ++ description='Copy a file from Ceph File System to Local Directory.') ++ get_parser.add_argument('remote_path', type=str, action=path_to_bytes, ++ help='Path of the file in the remote system') ++ get_parser.add_argument('local_path', type=str, action=path_to_bytes, ++ help='Path of the file in the local system') ++ get_parser.add_argument('-f', '--force', action='store_true', ++ help='Overwrites the destination if it already exists.') ++ ++ @with_argparser(get_parser) ++ def do_get(self, args): ++ """ ++ Copy a file/directory from CephFS to given path. ++ """ ++ if not is_file_exists(args.remote_path) and not \ ++ is_dir_exists(args.remote_path): ++ set_exit_code_msg(errno.ENOENT, "error: no file/directory" ++ " found at specified remote " ++ "path") ++ return ++ if (os.path.isfile(args.local_path) or os.path.isdir( ++ args.local_path)) and not args.force: ++ set_exit_code_msg(msg=f"error: file/directory " ++ f"{args.local_path.decode('utf-8')}" ++ f" already exists, use --force to " ++ f"overwrite") ++ return ++ root_src_dir = args.remote_path ++ root_dst_dir = args.local_path ++ fname = root_src_dir.rsplit(b'/', 1) ++ if args.local_path == b'.': ++ root_dst_dir = os.getcwdb() ++ if args.remote_path == b'.': ++ root_src_dir = cephfs.getcwd() ++ if args.local_path == b'-': ++ if args.remote_path == b'.' or args.remote_path == b'./': ++ set_exit_code_msg(errno.EINVAL, 'error: no remote file name specified') ++ return ++ copy_to_local(root_src_dir, b'-') ++ elif is_file_exists(args.remote_path): ++ copy_to_local(root_src_dir, root_dst_dir) ++ elif b'/' in root_src_dir and is_file_exists(fname[1], fname[0]): ++ copy_to_local(root_src_dir, root_dst_dir) ++ else: ++ files = list(reversed(sorted(dirwalk(root_src_dir)))) ++ for file_ in files: ++ dst_dirpath, dst_file = file_.rsplit(b'/', 1) ++ if dst_dirpath in files: ++ files.remove(dst_dirpath) ++ dst_path = os.path.join(root_dst_dir, dst_dirpath, dst_file) ++ dst_path = os.path.normpath(dst_path) ++ if is_dir_exists(file_): ++ try: ++ os.makedirs(dst_path) ++ except OSError: ++ pass ++ else: ++ copy_to_local(file_, dst_path) ++ ++ return 0 ++ ++ def complete_ls(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ ls_parser = argparse.ArgumentParser( ++ description='Copy a file from Ceph File System from Local Directory.') ++ ls_parser.add_argument('-l', '--long', action='store_true', ++ help='Detailed list of items in the directory.') ++ ls_parser.add_argument('-r', '--reverse', action='store_true', ++ help='Reverse order of listing items in the directory.') ++ ls_parser.add_argument('-H', action='store_true', help='Human Readable') ++ ls_parser.add_argument('-a', '--all', action='store_true', ++ help='Do not Ignore entries starting with .') ++ ls_parser.add_argument('-S', action='store_true', help='Sort by file_size') ++ ls_parser.add_argument('paths', help='Name of Directories', ++ action=path_to_bytes, nargs='*', default=['.']) ++ ++ @with_argparser(ls_parser) ++ def do_ls(self, args): ++ """ ++ List all the files and directories in the current working directory ++ """ ++ paths = args.paths ++ for path in paths: ++ values = [] ++ items = [] ++ try: ++ if path.count(b'*') > 0: ++ all_items = get_all_possible_paths(path) ++ if len(all_items) == 0: ++ continue ++ path = all_items[0].rsplit(b'/', 1)[0] ++ if path == b'': ++ path = b'/' ++ dirs = [] ++ for i in all_items: ++ for item in ls(path): ++ d_name = item.d_name ++ if os.path.basename(i) == d_name: ++ if item.is_dir(): ++ dirs.append(os.path.join(path, d_name)) ++ else: ++ items.append(item) ++ if dirs: ++ paths.extend(dirs) ++ else: ++ poutput(path.decode('utf-8'), end=':\n') ++ items = sorted(items, key=lambda item: item.d_name) ++ else: ++ if path != b'' and path != cephfs.getcwd() and len(paths) > 1: ++ poutput(path.decode('utf-8'), end=':\n') ++ items = sorted(ls(path), key=lambda item: item.d_name) ++ if not args.all: ++ items = [i for i in items if not i.d_name.startswith(b'.')] ++ if args.S: ++ items = sorted(items, key=lambda item: cephfs.stat( ++ path + b'/' + item.d_name, follow_symlink=( ++ not item.is_symbol_file())).st_size) ++ if args.reverse: ++ items = reversed(items) ++ for item in items: ++ filepath = item.d_name ++ is_dir = item.is_dir() ++ is_sym_lnk = item.is_symbol_file() ++ try: ++ if args.long and args.H: ++ print_long(os.path.join(cephfs.getcwd(), path, filepath), is_dir, ++ is_sym_lnk, True) ++ elif args.long: ++ print_long(os.path.join(cephfs.getcwd(), path, filepath), is_dir, ++ is_sym_lnk, False) ++ elif is_sym_lnk or is_dir: ++ values.append(style_listing(filepath.decode('utf-8'), is_dir, ++ is_sym_lnk)) ++ else: ++ values.append(filepath) ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ if not args.long: ++ print_list(values, shutil.get_terminal_size().columns) ++ if path != paths[-1]: ++ poutput('') ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ ++ def complete_rmdir(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ rmdir_parser = argparse.ArgumentParser(description='Remove Directory.') ++ rmdir_parser.add_argument('paths', help='Directory Path.', nargs='+', ++ action=path_to_bytes) ++ rmdir_parser.add_argument('-p', '--parent', action='store_true', ++ help='Remove parent directories as necessary. ' ++ 'When this option is specified, no error ' ++ 'is reported if a directory has any ' ++ 'sub-directories, files') ++ ++ @with_argparser(rmdir_parser) ++ def do_rmdir(self, args): ++ self.do_rmdir_helper(args) ++ ++ def do_rmdir_helper(self, args): ++ """ ++ Remove a specific Directory ++ """ ++ is_pattern = False ++ paths = args.paths ++ for path in paths: ++ if path.count(b'*') > 0: ++ is_pattern = True ++ all_items = get_all_possible_paths(path) ++ if len(all_items) > 0: ++ path = all_items[0].rsplit(b'/', 1)[0] ++ if path == b'': ++ path = b'/' ++ dirs = [] ++ for i in all_items: ++ for item in ls(path): ++ d_name = item.d_name ++ if os.path.basename(i) == d_name: ++ if item.is_dir(): ++ dirs.append(os.path.join(path, d_name)) ++ paths.extend(dirs) ++ continue ++ else: ++ is_pattern = False ++ ++ if args.parent: ++ path = os.path.join(cephfs.getcwd(), path.rsplit(b'/')[0]) ++ files = list(sorted(set(dirwalk(path)), reverse=True)) ++ if not files: ++ path = b'.' ++ for filepath in files: ++ try: ++ cephfs.rmdir(os.path.normpath(filepath)) ++ except libcephfs.Error as e: ++ perror(e) ++ path = b'.' ++ break ++ else: ++ path = os.path.normpath(os.path.join(cephfs.getcwd(), path)) ++ if not is_pattern and path != os.path.normpath(b''): ++ try: ++ cephfs.rmdir(path) ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ ++ def complete_rm(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ rm_parser = argparse.ArgumentParser(description='Remove File.') ++ rm_parser.add_argument('paths', help='File Path.', nargs='+', ++ action=path_to_bytes) ++ ++ @with_argparser(rm_parser) ++ def do_rm(self, args): ++ """ ++ Remove a specific file ++ """ ++ file_paths = args.paths ++ for path in file_paths: ++ if path.count(b'*') > 0: ++ file_paths.extend([i for i in get_all_possible_paths( ++ path) if is_file_exists(i)]) ++ else: ++ try: ++ cephfs.unlink(path) ++ except libcephfs.Error as e: ++ # NOTE: perhaps we need a better msg here ++ set_exit_code_msg(msg=e) ++ ++ def complete_mv(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ mv_parser = argparse.ArgumentParser(description='Move File.') ++ mv_parser.add_argument('src_path', type=str, action=path_to_bytes, ++ help='Source File Path.') ++ mv_parser.add_argument('dest_path', type=str, action=path_to_bytes, ++ help='Destination File Path.') ++ ++ @with_argparser(mv_parser) ++ def do_mv(self, args): ++ """ ++ Rename a file or Move a file from source path to the destination ++ """ ++ cephfs.rename(args.src_path, args.dest_path) ++ ++ def complete_cd(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ cd_parser = argparse.ArgumentParser(description='Change working directory') ++ cd_parser.add_argument('path', type=str, help='Name of the directory.', ++ action=path_to_bytes, nargs='?', default='/') ++ ++ @with_argparser(cd_parser) ++ def do_cd(self, args): ++ """ ++ Change working directory ++ """ ++ cephfs.chdir(args.path) ++ self.working_dir = cephfs.getcwd().decode('utf-8') ++ self.set_prompt() ++ ++ def do_cwd(self, arglist): ++ """ ++ Get current working directory. ++ """ ++ poutput(cephfs.getcwd().decode('utf-8')) ++ ++ def complete_chmod(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ chmod_parser = argparse.ArgumentParser(description='Create Directory.') ++ chmod_parser.add_argument('mode', type=str, action=ModeAction, help='Mode') ++ chmod_parser.add_argument('paths', type=str, action=path_to_bytes, ++ help='Name of the file', nargs='+') ++ ++ @with_argparser(chmod_parser) ++ def do_chmod(self, args): ++ """ ++ Change permission of a file ++ """ ++ for path in args.paths: ++ mode = int(args.mode, base=8) ++ try: ++ cephfs.chmod(path, mode) ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ ++ def complete_cat(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ cat_parser = argparse.ArgumentParser(description='') ++ cat_parser.add_argument('paths', help='Name of Files', action=path_to_bytes, ++ nargs='+') ++ ++ @with_argparser(cat_parser) ++ def do_cat(self, args): ++ """ ++ Print contents of a file ++ """ ++ for path in args.paths: ++ if is_file_exists(path): ++ copy_to_local(path, b'-') ++ else: ++ set_exit_code_msg(errno.ENOENT, '{}: no such file'.format( ++ path.decode('utf-8'))) ++ ++ umask_parser = argparse.ArgumentParser(description='Set umask value.') ++ umask_parser.add_argument('mode', help='Mode', type=str, action=ModeAction, ++ nargs='?', default='') ++ ++ @with_argparser(umask_parser) ++ def do_umask(self, args): ++ """ ++ Set Umask value. ++ """ ++ if args.mode == '': ++ poutput(self.umask.zfill(4)) ++ else: ++ mode = int(args.mode, 8) ++ self.umask = str(oct(cephfs.umask(mode))[2:]) ++ ++ def complete_write(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ write_parser = argparse.ArgumentParser(description='Writes data into a file') ++ write_parser.add_argument('path', type=str, action=path_to_bytes, ++ help='Name of File') ++ ++ @with_argparser(write_parser) ++ def do_write(self, args): ++ """ ++ Write data into a file. ++ """ ++ ++ copy_from_local(b'-', args.path) ++ ++ def complete_lcd(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ index_dict = {1: self.path_complete} ++ return self.index_based_complete(text, line, begidx, endidx, index_dict) ++ ++ lcd_parser = argparse.ArgumentParser(description='') ++ lcd_parser.add_argument('path', type=str, action=path_to_bytes, help='Path') ++ ++ @with_argparser(lcd_parser) ++ def do_lcd(self, args): ++ """ ++ Moves into the given local directory ++ """ ++ try: ++ os.chdir(os.path.expanduser(args.path)) ++ except OSError as e: ++ set_exit_code_msg(e.errno, "Cannot change to " ++ f"{e.filename.decode('utf-8')}: {e.strerror}") ++ ++ def complete_lls(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ index_dict = {1: self.path_complete} ++ return self.index_based_complete(text, line, begidx, endidx, index_dict) ++ ++ lls_parser = argparse.ArgumentParser( ++ description='List files in local system.') ++ lls_parser.add_argument('paths', help='Paths', action=path_to_bytes, ++ nargs='*') ++ ++ @with_argparser(lls_parser) ++ def do_lls(self, args): ++ """ ++ Lists all files and folders in the current local directory ++ """ ++ if not args.paths: ++ print_list(os.listdir(os.getcwdb())) ++ else: ++ for path in args.paths: ++ try: ++ items = os.listdir(path) ++ poutput("{}:".format(path.decode('utf-8'))) ++ print_list(items) ++ except OSError as e: ++ set_exit_code_msg(e.errno, f"{e.filename.decode('utf-8')}: " ++ f"{e.strerror}") ++ # Arguments to the with_argpaser decorator function are sticky. ++ # The items in args.path do not get overwritten in subsequent calls. ++ # The arguments remain in args.paths after the function exits and we ++ # neeed to clean it up to ensure the next call works as expected. ++ args.paths.clear() ++ ++ def do_lpwd(self, arglist): ++ """ ++ Prints the absolute path of the current local directory ++ """ ++ poutput(os.getcwd()) ++ ++ def complete_df(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ df_parser = argparse.ArgumentParser(description='Show information about\ ++ the amount of available disk space') ++ df_parser.add_argument('file', help='Name of the file', nargs='*', ++ default=['.'], action=path_to_bytes) ++ ++ @with_argparser(df_parser) ++ def do_df(self, arglist): ++ """ ++ Display the amount of available disk space for file systems ++ """ ++ header = True # Set to true for printing header only once ++ if b'.' == arglist.file[0]: ++ arglist.file = ls(b'.') ++ ++ for file in arglist.file: ++ if isinstance(file, libcephfs.DirEntry): ++ file = file.d_name ++ if file == b'.' or file == b'..': ++ continue ++ try: ++ statfs = cephfs.statfs(file) ++ stat = cephfs.stat(file) ++ block_size = (statfs['f_blocks'] * statfs['f_bsize']) // 1024 ++ available = block_size - stat.st_size ++ use = 0 ++ ++ if block_size > 0: ++ use = (stat.st_size * 100) // block_size ++ ++ if header: ++ header = False ++ poutput('{:25s}\t{:5s}\t{:15s}{:10s}{}'.format( ++ "1K-blocks", "Used", "Available", "Use%", ++ "Stored on")) ++ ++ poutput('{:d}\t{:18d}\t{:8d}\t{:10s} {}'.format(block_size, ++ stat.st_size, available, str(int(use)) + '%', ++ file.decode('utf-8'))) ++ except libcephfs.OSError as e: ++ set_exit_code_msg(e.get_error_code(), "could not statfs {}: {}".format( ++ file.decode('utf-8'), e.strerror)) ++ ++ locate_parser = argparse.ArgumentParser( ++ description='Find file within file system') ++ locate_parser.add_argument('name', help='name', type=str, ++ action=path_to_bytes) ++ locate_parser.add_argument('-c', '--count', action='store_true', ++ help='Count list of items located.') ++ locate_parser.add_argument( ++ '-i', '--ignorecase', action='store_true', help='Ignore case') ++ ++ @with_argparser(locate_parser) ++ def do_locate(self, args): ++ """ ++ Find a file within the File System ++ """ ++ if args.name.count(b'*') == 1: ++ if args.name[0] == b'*': ++ args.name += b'/' ++ elif args.name[-1] == '*': ++ args.name = b'/' + args.name ++ args.name = args.name.replace(b'*', b'') ++ if args.ignorecase: ++ locations = locate_file(args.name, False) ++ else: ++ locations = locate_file(args.name) ++ if args.count: ++ poutput(len(locations)) ++ else: ++ poutput((b'\n'.join(locations)).decode('utf-8')) ++ ++ def complete_du(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ du_parser = argparse.ArgumentParser( ++ description='Disk Usage of a Directory') ++ du_parser.add_argument('paths', type=str, action=get_list_of_bytes_path, ++ help='Name of the directory.', nargs='*', ++ default=[b'.']) ++ du_parser.add_argument('-r', action='store_true', ++ help='Recursive Disk usage of all directories.') ++ ++ @with_argparser(du_parser) ++ def do_du(self, args): ++ """ ++ Print disk usage of a given path(s). ++ """ ++ def print_disk_usage(files): ++ if isinstance(files, bytes): ++ files = (files, ) ++ ++ for f in files: ++ try: ++ st = cephfs.lstat(f) ++ ++ if stat.S_ISDIR(st.st_mode): ++ dusage = int(cephfs.getxattr(f, ++ 'ceph.dir.rbytes').decode('utf-8')) ++ else: ++ dusage = st.st_size ++ ++ # print path in local context ++ f = os.path.normpath(f) ++ if f[0] is ord('/'): ++ f = b'.' + f ++ poutput('{:10s} {}'.format(humansize(dusage), ++ f.decode('utf-8'))) ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ continue ++ ++ for path in args.paths: ++ if args.r: ++ print_disk_usage(sorted(set(dirwalk(path)).union({path}))) ++ else: ++ print_disk_usage(path) ++ ++ quota_parser = argparse.ArgumentParser( ++ description='Quota management for a Directory') ++ quota_parser.add_argument('op', choices=['get', 'set'], ++ help='Quota operation type.') ++ quota_parser.add_argument('path', type=str, action=path_to_bytes, ++ help='Name of the directory.') ++ quota_parser.add_argument('--max_bytes', type=int, default=-1, nargs='?', ++ help='Max cumulative size of the data under ' ++ 'this directory.') ++ quota_parser.add_argument('--max_files', type=int, default=-1, nargs='?', ++ help='Total number of files under this ' ++ 'directory tree.') ++ ++ @with_argparser(quota_parser) ++ def do_quota(self, args): ++ """ ++ Quota management. ++ """ ++ if not is_dir_exists(args.path): ++ set_exit_code_msg(errno.ENOENT, 'error: no such directory {}'.format( ++ args.path.decode('utf-8'))) ++ return ++ ++ if args.op == 'set': ++ if (args.max_bytes == -1) and (args.max_files == -1): ++ set_exit_code_msg(errno.EINVAL, 'please specify either ' ++ '--max_bytes or --max_files or both') ++ return ++ ++ if args.max_bytes >= 0: ++ max_bytes = to_bytes(str(args.max_bytes)) ++ try: ++ cephfs.setxattr(args.path, 'ceph.quota.max_bytes', ++ max_bytes, os.XATTR_CREATE) ++ poutput('max_bytes set to %d' % args.max_bytes) ++ except libcephfs.Error as e: ++ cephfs.setxattr(args.path, 'ceph.quota.max_bytes', ++ max_bytes, os.XATTR_REPLACE) ++ set_exit_code_msg(e.get_error_code(), 'max_bytes reset to ' ++ f'{args.max_bytes}') ++ ++ if args.max_files >= 0: ++ max_files = to_bytes(str(args.max_files)) ++ try: ++ cephfs.setxattr(args.path, 'ceph.quota.max_files', ++ max_files, os.XATTR_CREATE) ++ poutput('max_files set to %d' % args.max_files) ++ except libcephfs.Error as e: ++ cephfs.setxattr(args.path, 'ceph.quota.max_files', ++ max_files, os.XATTR_REPLACE) ++ set_exit_code_msg(e.get_error_code(), 'max_files reset to ' ++ f'{args.max_files}') ++ elif args.op == 'get': ++ max_bytes = '0' ++ max_files = '0' ++ try: ++ max_bytes = cephfs.getxattr(args.path, 'ceph.quota.max_bytes') ++ poutput('max_bytes: {}'.format(max_bytes.decode('utf-8'))) ++ except libcephfs.Error as e: ++ set_exit_code_msg(e.get_error_code(), 'max_bytes is not set') ++ ++ try: ++ max_files = cephfs.getxattr(args.path, 'ceph.quota.max_files') ++ poutput('max_files: {}'.format(max_files.decode('utf-8'))) ++ except libcephfs.Error as e: ++ set_exit_code_msg(e.get_error_code(), 'max_files is not set') ++ ++ snap_parser = argparse.ArgumentParser(description='Snapshot Management') ++ snap_parser.add_argument('op', type=str, ++ help='Snapshot operation: create or delete') ++ snap_parser.add_argument('name', type=str, action=path_to_bytes, ++ help='Name of snapshot') ++ snap_parser.add_argument('dir', type=str, action=path_to_bytes, ++ help='Directory for which snapshot ' ++ 'needs to be created or deleted') ++ ++ @with_argparser(snap_parser) ++ def do_snap(self, args): ++ """ ++ Snapshot management for the volume ++ """ ++ # setting self.colors to None turns off colorizing and ++ # perror emits plain text ++ self.colors = None ++ ++ snapdir = '.snap' ++ conf_snapdir = cephfs.conf_get('client_snapdir') ++ if conf_snapdir is not None: ++ snapdir = conf_snapdir ++ snapdir = to_bytes(snapdir) ++ if args.op == 'create': ++ try: ++ if is_dir_exists(args.dir): ++ cephfs.mkdir(os.path.join(args.dir, snapdir, args.name), 0o755) ++ else: ++ set_exit_code_msg(errno.ENOENT, "'{}': no such directory".format( ++ args.dir.decode('utf-8'))) ++ except libcephfs.Error as e: ++ set_exit_code_msg(e.get_error_code(), ++ "snapshot '{}' already exists".format( ++ args.name.decode('utf-8'))) ++ elif args.op == 'delete': ++ snap_dir = os.path.join(args.dir, snapdir, args.name) ++ try: ++ if is_dir_exists(snap_dir): ++ newargs = argparse.Namespace(paths=[snap_dir], parent=False) ++ self.do_rmdir_helper(newargs) ++ else: ++ set_exit_code_msg(errno.ENOENT, "'{}': no such snapshot".format( ++ args.name.decode('utf-8'))) ++ except libcephfs.Error as e: ++ set_exit_code_msg(e.get_error_code(), "error while deleting " ++ "'{}'".format(snap_dir.decode('utf-8'))) ++ else: ++ set_exit_code_msg(errno.EINVAL, "snapshot can only be created or " ++ "deleted; check - help snap") ++ ++ def do_help(self, line): ++ """ ++ Get details about a command. ++ Usage: help - for a specific command ++ help all - for all the commands ++ """ ++ if line == 'all': ++ for k in dir(self): ++ if k.startswith('do_'): ++ poutput('-' * 80) ++ super().do_help(k[3:]) ++ return ++ parser = self.create_argparser(line) ++ if parser: ++ parser.print_help() ++ else: ++ super().do_help(line) ++ ++ def complete_stat(self, text, line, begidx, endidx): ++ """ ++ auto complete of file name. ++ """ ++ return self.complete_filenames(text, line, begidx, endidx) ++ ++ stat_parser = argparse.ArgumentParser( ++ description='Display file or file system status') ++ stat_parser.add_argument('paths', type=str, help='file paths', ++ action=path_to_bytes, nargs='+') ++ ++ @with_argparser(stat_parser) ++ def do_stat(self, args): ++ """ ++ Display file or file system status ++ """ ++ for path in args.paths: ++ try: ++ stat = cephfs.stat(path) ++ atime = stat.st_atime.isoformat(' ') ++ mtime = stat.st_mtime.isoformat(' ') ++ ctime = stat.st_mtime.isoformat(' ') ++ ++ poutput("File: {}\nSize: {:d}\nBlocks: {:d}\nIO Block: {:d}\n" ++ "Device: {:d}\tInode: {:d}\tLinks: {:d}\nPermission: " ++ "{:o}/{}\tUid: {:d}\tGid: {:d}\nAccess: {}\nModify: " ++ "{}\nChange: {}".format(path.decode('utf-8'), ++ stat.st_size, stat.st_blocks, ++ stat.st_blksize, stat.st_dev, ++ stat.st_ino, stat.st_nlink, ++ stat.st_mode, ++ mode_notation(stat.st_mode), ++ stat.st_uid, stat.st_gid, atime, ++ mtime, ctime)) ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ ++ setxattr_parser = argparse.ArgumentParser( ++ description='Set extended attribute for a file') ++ setxattr_parser.add_argument('path', type=str, action=path_to_bytes, help='Name of the file') ++ setxattr_parser.add_argument('name', type=str, help='Extended attribute name') ++ setxattr_parser.add_argument('value', type=str, help='Extended attribute value') ++ ++ @with_argparser(setxattr_parser) ++ def do_setxattr(self, args): ++ """ ++ Set extended attribute for a file ++ """ ++ val_bytes = to_bytes(args.value) ++ name_bytes = to_bytes(args.name) ++ try: ++ cephfs.setxattr(args.path, name_bytes, val_bytes, os.XATTR_CREATE) ++ poutput('{} is successfully set to {}'.format(args.name, args.value)) ++ except libcephfs.ObjectExists: ++ cephfs.setxattr(args.path, name_bytes, val_bytes, os.XATTR_REPLACE) ++ poutput('{} is successfully reset to {}'.format(args.name, args.value)) ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ ++ getxattr_parser = argparse.ArgumentParser( ++ description='Get extended attribute set for a file') ++ getxattr_parser.add_argument('path', type=str, action=path_to_bytes, ++ help='Name of the file') ++ getxattr_parser.add_argument('name', type=str, help='Extended attribute name') ++ ++ @with_argparser(getxattr_parser) ++ def do_getxattr(self, args): ++ """ ++ Get extended attribute for a file ++ """ ++ try: ++ poutput('{}'.format(cephfs.getxattr(args.path, ++ to_bytes(args.name)).decode('utf-8'))) ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ ++ listxattr_parser = argparse.ArgumentParser( ++ description='List extended attributes set for a file') ++ listxattr_parser.add_argument('path', type=str, action=path_to_bytes, ++ help='Name of the file') ++ ++ @with_argparser(listxattr_parser) ++ def do_listxattr(self, args): ++ """ ++ List extended attributes for a file ++ """ ++ try: ++ size, xattr_list = cephfs.listxattr(args.path) ++ if size > 0: ++ poutput('{}'.format(xattr_list.replace(b'\x00', b' ').decode('utf-8'))) ++ else: ++ poutput('No extended attribute is set') ++ except libcephfs.Error as e: ++ set_exit_code_msg(msg=e) ++ ++ ++####################################################### ++# ++# Following are methods that get cephfs-shell started. ++# ++##################################################### ++ ++def setup_cephfs(args): ++ """ ++ Mounting a cephfs ++ """ ++ global cephfs ++ try: ++ cephfs = libcephfs.LibCephFS(conffile='') ++ cephfs.mount(filesystem_name=args.fs) ++ except libcephfs.ObjectNotFound as e: ++ print('couldn\'t find ceph configuration not found') ++ sys.exit(e.get_error_code()) ++ except libcephfs.Error as e: ++ print(e) ++ sys.exit(e.get_error_code()) ++ ++ ++def str_to_bool(val): ++ """ ++ Return corresponding bool values for strings like 'true' or 'false'. ++ """ ++ if not isinstance(val, str): ++ return val ++ ++ val = val.replace('\n', '') ++ if val.lower() in ['true', 'yes']: ++ return True ++ elif val.lower() in ['false', 'no']: ++ return False ++ else: ++ return val ++ ++ ++def read_shell_conf(shell, shell_conf_file): ++ import configparser ++ ++ sec = 'cephfs-shell' ++ opts = [] ++ if LooseVersion(cmd2_version) >= LooseVersion("0.10.0"): ++ for attr in shell.settables.keys(): ++ opts.append(attr) ++ else: ++ if LooseVersion(cmd2_version) <= LooseVersion("0.9.13"): ++ # hardcoding options for 0.7.9 because - ++ # 1. we use cmd2 v0.7.9 with teuthology and ++ # 2. there's no way distinguish between a shell setting and shell ++ # object attribute until v0.10.0 ++ opts = ['abbrev', 'autorun_on_edit', 'colors', ++ 'continuation_prompt', 'debug', 'echo', 'editor', ++ 'feedback_to_output', 'locals_in_py', 'prompt', 'quiet', ++ 'timing'] ++ elif LooseVersion(cmd2_version) >= LooseVersion("0.9.23"): ++ opts.append('allow_style') ++ # no equivalent option was defined by cmd2. ++ else: ++ pass ++ ++ # default and only section in our conf file. ++ cp = configparser.ConfigParser(default_section=sec, strict=False) ++ cp.read(shell_conf_file) ++ for opt in opts: ++ if cp.has_option(sec, opt): ++ setattr(shell, opt, str_to_bool(cp.get(sec, opt))) ++ ++ ++def get_shell_conffile_path(arg_conf=''): ++ conf_filename = 'cephfs-shell.conf' ++ env_var = 'CEPHFS_SHELL_CONF' ++ ++ arg_conf = '' if not arg_conf else arg_conf ++ home_dir_conf = os.path.expanduser('~/.' + conf_filename) ++ env_conf = os.environ[env_var] if env_var in os.environ else '' ++ ++ # here's the priority by which conf gets read. ++ for path in (arg_conf, env_conf, home_dir_conf): ++ if os.path.isfile(path): ++ return path ++ else: ++ return '' ++ ++ ++def manage_args(): ++ main_parser = argparse.ArgumentParser(description='') ++ main_parser.add_argument('-b', '--batch', action='store', ++ help='Path to CephFS shell script/batch file' ++ 'containing CephFS shell commands', ++ type=str) ++ main_parser.add_argument('-c', '--config', action='store', ++ help='Path to Ceph configuration file.', ++ type=str) ++ main_parser.add_argument('-f', '--fs', action='store', ++ help='Name of filesystem to mount.', ++ type=str) ++ main_parser.add_argument('-t', '--test', action='store', ++ help='Test against transcript(s) in FILE', ++ nargs='+') ++ main_parser.add_argument('commands', nargs='*', help='Comma delimited ' ++ 'commands. The shell executes the given command ' ++ 'and quits immediately with the return value of ' ++ 'command. In case no commands are provided, the ' ++ 'shell is launched.', default=[]) ++ ++ args = main_parser.parse_args() ++ args.exe_and_quit = False # Execute and quit, don't launch the shell. ++ ++ if args.batch: ++ if LooseVersion(cmd2_version) <= LooseVersion("0.9.13"): ++ args.commands = ['load ' + args.batch, ',quit'] ++ else: ++ args.commands = ['run_script ' + args.batch, ',quit'] ++ if args.test: ++ args.commands.extend(['-t,'] + [arg + ',' for arg in args.test]) ++ if not args.batch and len(args.commands) > 0: ++ args.exe_and_quit = True ++ ++ manage_sys_argv(args) ++ ++ return args ++ ++ ++def manage_sys_argv(args): ++ exe = sys.argv[0] ++ sys.argv.clear() ++ sys.argv.append(exe) ++ sys.argv.extend([i.strip() for i in ' '.join(args.commands).split(',')]) ++ ++ setup_cephfs(args) ++ ++ ++def execute_cmd_args(args): ++ """ ++ Launch a shell session if no arguments were passed, else just execute ++ the given argument as a shell command and exit the shell session ++ immediately at (last) command's termination with the (last) command's ++ return value. ++ """ ++ if not args.exe_and_quit: ++ return shell.cmdloop() ++ return execute_cmds_and_quit(args) ++ ++ ++def execute_cmds_and_quit(args): ++ """ ++ Multiple commands might be passed separated by commas, feed onecmd() ++ one command at a time. ++ """ ++ # do_* methods triggered by cephfs-shell commands return None when they ++ # complete running successfully. Until 0.9.6, shell.onecmd() returned this ++ # value to indicate whether the execution of the commands should stop, but ++ # since 0.9.7 it returns the return value of do_* methods only if it's ++ # not None. When it is None it returns False instead of None. ++ if LooseVersion(cmd2_version) <= LooseVersion("0.9.6"): ++ stop_exec_val = None ++ else: ++ stop_exec_val = False ++ ++ args_to_onecmd = '' ++ if len(args.commands) <= 1: ++ args.commands = args.commands[0].split(' ') ++ for cmdarg in args.commands: ++ if ',' in cmdarg: ++ args_to_onecmd += ' ' + cmdarg[0:-1] ++ onecmd_retval = shell.onecmd(args_to_onecmd) ++ # if the curent command failed, let's abort the execution of ++ # series of commands passed. ++ if onecmd_retval is not stop_exec_val: ++ return onecmd_retval ++ if shell.exit_code != 0: ++ return shell.exit_code ++ ++ args_to_onecmd = '' ++ continue ++ ++ args_to_onecmd += ' ' + cmdarg ++ return shell.onecmd(args_to_onecmd) ++ ++ ++if __name__ == '__main__': ++ args = manage_args() ++ ++ shell = CephFSShell() ++ # TODO: perhaps, we should add an option to pass ceph.conf? ++ read_shell_conf(shell, get_shell_conffile_path(args.config)) ++ # XXX: setting shell.exit_code to zero so that in case there are no errors ++ # and exceptions, it is not set by any method or function of cephfs-shell ++ # and return values from shell.cmdloop() or shell.onecmd() is not an ++ # integer, we can treat it as the return value of cephfs-shell. ++ shell.exit_code = 0 ++ ++ retval = execute_cmd_args(args) ++ sys.exit(retval if retval else shell.exit_code) +--- /dev/null 2022-06-30 09:45:32.996000000 -0400 ++++ ceph-17.2.1/src/tools/cephfs/shell/setup.py 2022-07-05 11:00:12.411260682 -0400 +@@ -0,0 +1,27 @@ ++# -*- coding: utf-8 -*- ++ ++from setuptools import setup ++ ++__version__ = '0.0.1' ++ ++setup( ++ name='cephfs-shell', ++ version=__version__, ++ description='Interactive shell for Ceph file system', ++ keywords='cephfs, shell', ++ scripts=['cephfs-shell'], ++ install_requires=[ ++ 'cephfs', ++ 'cmd2', ++ 'colorama', ++ ], ++ classifiers=[ ++ 'Development Status :: 3 - Alpha', ++ 'Environment :: Console', ++ 'Intended Audience :: System Administrators', ++ 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)', ++ 'Operating System :: POSIX :: Linux', ++ 'Programming Language :: Python :: 3' ++ ], ++ license='LGPLv2+', ++) +--- /dev/null 2022-06-30 09:45:32.996000000 -0400 ++++ ceph-17.2.1/src/tools/cephfs/shell/tox.ini 2022-06-23 10:41:35.000000000 -0400 +@@ -0,0 +1,7 @@ ++[tox] ++envlist = py3 ++skipsdist = true ++ ++[testenv:py3] ++deps = flake8 ++commands = flake8 --ignore=W503 --max-line-length=100 cephfs-shell diff --git a/0021-src-rgw-CMakeLists.txt.patch b/0021-src-rgw-CMakeLists.txt.patch deleted file mode 100644 index 48e7ed3550961f17292c8f72b19cc5026afa0161..0000000000000000000000000000000000000000 --- a/0021-src-rgw-CMakeLists.txt.patch +++ /dev/null @@ -1,11 +0,0 @@ ---- ceph-17.2.0/src/rgw/CMakeLists.txt.orig 2022-04-28 13:44:40.547580835 -0400 -+++ ceph-17.2.0/src/rgw/CMakeLists.txt 2022-04-28 13:45:05.040153926 -0400 -@@ -4,7 +4,7 @@ - endif() - - if(WITH_RADOSGW_SELECT_PARQUET) -- set(ARROW_LIBRARIES Arrow::Parquet) -+ set(ARROW_LIBRARIES Arrow::Arrow Arrow::Parquet) - add_definitions(-D_ARROW_EXIST) - message("-- arrow is installed, radosgw/s3select-op is able to process parquet objects") - endif(WITH_RADOSGW_SELECT_PARQUET) diff --git a/0022-mon-Replace-deprecated-use-of-format_to.patch b/0022-mon-Replace-deprecated-use-of-format_to.patch new file mode 100644 index 0000000000000000000000000000000000000000..76edaff61a5b291d86db0b4e2fbe53e500b255c7 --- /dev/null +++ b/0022-mon-Replace-deprecated-use-of-format_to.patch @@ -0,0 +1,28 @@ +From fff72cd14c58d06774cbd0274e6144b42448af03 Mon Sep 17 00:00:00 2001 +From: "Adam C. Emerson" +Date: Mon, 7 Mar 2022 18:54:30 -0500 +Subject: [PATCH] mon: Replace deprecated use of format_to + +The non-deprecated version takes an explicit OutputIterator. + +Signed-off-by: Adam C. Emerson +--- + src/mon/LogMonitor.cc | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/src/mon/LogMonitor.cc b/src/mon/LogMonitor.cc +index 9103ddf7c5b..c196e8429fb 100644 +--- a/src/mon/LogMonitor.cc ++++ b/src/mon/LogMonitor.cc +@@ -411,7 +411,7 @@ void LogMonitor::log_external(const LogEntry& le) + } + + if (fd >= 0) { +- fmt::format_to(file_log_buffer, "{}\n", le); ++ fmt::format_to(std::back_inserter(file_log_buffer), "{}\n", le); + int err = safe_write(fd, file_log_buffer.data(), file_log_buffer.size()); + file_log_buffer.clear(); + if (err < 0) { +-- +2.36.1 + diff --git a/ceph-17.2.0.tar.gz b/ceph-17.2.3.tar.gz similarity index 89% rename from ceph-17.2.0.tar.gz rename to ceph-17.2.3.tar.gz index 6cad0daa2891721b13d0182bc5fabaaf3459e78f..b14e0b40bb2105e181ff956cc5a3a3a0b9bd19c9 100644 Binary files a/ceph-17.2.0.tar.gz and b/ceph-17.2.3.tar.gz differ diff --git a/ceph.spec b/ceph.spec index 3bb470e5d7f2653a16e4102b66114dfd5367e568..26aebbc8bfe3d722b304cabff54e01885b73b45b 100644 --- a/ceph.spec +++ b/ceph.spec @@ -11,7 +11,6 @@ %bcond_without rbd_rwl_cache %else %bcond_with rbd_rwl_cache -%global _system_pmdk 1 %endif %ifarch %{arm64} %bcond_with system_pmdk @@ -68,7 +67,7 @@ # main package definition ################################################################################# Name: ceph -Version: 17.2.0 +Version: 17.2.3 Release: %{anolis_release}%{?dist} Epoch: 2 @@ -92,7 +91,8 @@ Patch0017: 0017-gcc-12-omnibus.patch Patch0018: 0018-src-rgw-store-dbstore-CMakeLists.txt.patch Patch0019: 0019-cmake-modules-CheckCxxAtomic.cmake.patch Patch0020: 0020-src-arrow-cpp-cmake_modules-ThirdpartyToolchain.cmake.patch -Patch0021: 0021-src-rgw-CMakeLists.txt.patch +Patch0021: 0021-cephfs-shell.patch +Patch0022: 0022-mon-Replace-deprecated-use-of-format_to.patch ################################################################################# # dependencies that apply across all distro families ################################################################################# @@ -174,7 +174,6 @@ BuildRequires: hostname BuildRequires: jq BuildRequires: libuuid-devel BuildRequires: python%{python3_pkgversion}-bcrypt -BuildRequires: python%{python3_pkgversion}-nose BuildRequires: python%{python3_pkgversion}-pecan BuildRequires: python%{python3_pkgversion}-requests BuildRequires: python%{python3_pkgversion}-dateutil @@ -216,6 +215,7 @@ BuildRequires: nss-devel BuildRequires: keyutils-libs-devel BuildRequires: libibverbs-devel BuildRequires: librdmacm-devel +BuildRequires: ninja-build BuildRequires: openldap-devel BuildRequires: openssl-devel BuildRequires: CUnit-devel @@ -432,6 +432,7 @@ Summary: Ceph Manager module for cephadm-based orchestration BuildArch: noarch Requires: ceph-mgr = %{_epoch_prefix}%{version}-%{release} Requires: python%{python3_pkgversion}-asyncssh +Requires: python%{python3_pkgversion}-natsort Requires: cephadm = %{_epoch_prefix}%{version}-%{release} Requires: openssh-clients Requires: python%{python3_pkgversion}-cherrypy @@ -1100,15 +1101,14 @@ getent passwd cephadm >/dev/null || useradd -r -g cephadm -s /bin/bash -c "cepha exit 0 %postun -n cephadm -userdel -r cephadm || true -exit 0 +[ $1 -ne 0 ] || userdel cephadm || : %files -n cephadm %{_sbindir}/cephadm %{_mandir}/man8/cephadm.8* %attr(0700,cephadm,cephadm) %dir %{_sharedstatedir}/cephadm %attr(0700,cephadm,cephadm) %dir %{_sharedstatedir}/cephadm/.ssh -%attr(0600,cephadm,cephadm) %{_sharedstatedir}/cephadm/.ssh/authorized_keys +%config(noreplace) %attr(0600,cephadm,cephadm) %{_sharedstatedir}/cephadm/.ssh/authorized_keys %files common %dir %{_docdir}/ceph @@ -1223,6 +1223,7 @@ fi %dir %{_datadir}/ceph/mgr %{_datadir}/ceph/mgr/mgr_module.* %{_datadir}/ceph/mgr/mgr_util.* +%{_datadir}/ceph/mgr/object_format.* %{_unitdir}/ceph-mgr@.service %{_unitdir}/ceph-mgr.target %attr(750,ceph,ceph) %dir %{_localstatedir}/lib/ceph/mgr @@ -1925,5 +1926,8 @@ exit 0 %config %{_sysconfdir}/prometheus/ceph/ceph_default_alerts.yml %changelog +* Wed Aug 31 2022 mgb01105731 - 17.2.3-1 +- Update ceph to 17.2.3 + * Tue May 17 2022 Chunmei Xu - 17.2.0-1 - init from upstream