diff --git a/src/main/java/com/hy/java/uct/cdtocode/CDToCodeTracer.java b/src/main/java/com/hy/java/uct/cdtocode/CDToCodeTracer.java index 96c071f2955ba5328f90f4a679c8971ab85d46ab..100989ac767a20fccc1fe2ce9fb410e5b494d543 100644 --- a/src/main/java/com/hy/java/uct/cdtocode/CDToCodeTracer.java +++ b/src/main/java/com/hy/java/uct/cdtocode/CDToCodeTracer.java @@ -10,7 +10,7 @@ import com.hy.java.uct.cdtocode.mapper.DocAnalyzer; import com.hy.java.uct.cdtocode.reader.CDReader; import com.hy.java.uct.cdtocode.reader.CodeReader; import com.hy.java.uct.cdtocode.reader.DocReader; -import com.hy.java.uct.util.UMLClass; +import com.hy.java.uct.util.cd.UMLClass; public class CDToCodeTracer { /** diff --git a/src/main/java/com/hy/java/uct/cdtocode/mapper/CodeRelationMapper.java b/src/main/java/com/hy/java/uct/cdtocode/mapper/CodeRelationMapper.java index 21c62168d293e579348d143c5829758c686df9ec..a15baf91b322eb31f6fe59c20b424f7557843c1b 100644 --- a/src/main/java/com/hy/java/uct/cdtocode/mapper/CodeRelationMapper.java +++ b/src/main/java/com/hy/java/uct/cdtocode/mapper/CodeRelationMapper.java @@ -27,8 +27,8 @@ import com.github.javaparser.ast.body.MethodDeclaration; import com.github.javaparser.ast.type.ClassOrInterfaceType; import com.hy.java.uct.cdtocode.util.Entity; import com.hy.java.uct.cdtocode.util.MappedFile; -import com.hy.java.uct.util.EntityRelation; -import com.hy.java.uct.util.UMLClass; +import com.hy.java.uct.util.cd.EntityRelation; +import com.hy.java.uct.util.cd.UMLClass; import com.hy.java.utility.common.Pair; import jxl.Sheet; diff --git a/src/main/java/com/hy/java/uct/cdtocode/mapper/DocAnalyzer.java b/src/main/java/com/hy/java/uct/cdtocode/mapper/DocAnalyzer.java index 26ba29eb0e13b0192699fb42bcb23f1591c5f27a..583d6f1f9cf02dfd22095e23f0326bcfb9dc67a8 100644 --- a/src/main/java/com/hy/java/uct/cdtocode/mapper/DocAnalyzer.java +++ b/src/main/java/com/hy/java/uct/cdtocode/mapper/DocAnalyzer.java @@ -13,9 +13,9 @@ import java.util.stream.Collectors; import org.apache.commons.collections4.SetUtils; import com.hy.java.uct.cdtocode.util.Entity; -import com.hy.java.uct.util.EntityRelation; -import com.hy.java.uct.util.ImgRelation; -import com.hy.java.uct.util.UMLClass; +import com.hy.java.uct.util.cd.EntityRelation; +import com.hy.java.uct.util.cd.ImgRelation; +import com.hy.java.uct.util.cd.UMLClass; import com.hy.java.utility.common.FileEditor; import com.hy.java.utility.common.Pair; diff --git a/src/main/java/com/hy/java/uct/cdtocode/reader/CDReader.java b/src/main/java/com/hy/java/uct/cdtocode/reader/CDReader.java index f39bc272eefd50c9abeebac4d5c4075e1616ce13..50da51674af914333b12685c0053fb2081b8ef68 100644 --- a/src/main/java/com/hy/java/uct/cdtocode/reader/CDReader.java +++ b/src/main/java/com/hy/java/uct/cdtocode/reader/CDReader.java @@ -4,8 +4,8 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; -import com.hy.java.uct.util.ImgRelation; -import com.hy.java.uct.util.UMLClass; +import com.hy.java.uct.util.cd.ImgRelation; +import com.hy.java.uct.util.cd.UMLClass; import com.hy.java.utility.common.FileEditor; public class CDReader { diff --git a/src/main/java/com/hy/java/uct/cdtocode/util/Entity.java b/src/main/java/com/hy/java/uct/cdtocode/util/Entity.java index 77f5338336fa6fed9bbf2356e165f97aafee6535..391781cbb70819164e38fa9aa06c2f02f48f4cff 100644 --- a/src/main/java/com/hy/java/uct/cdtocode/util/Entity.java +++ b/src/main/java/com/hy/java/uct/cdtocode/util/Entity.java @@ -3,8 +3,8 @@ package com.hy.java.uct.cdtocode.util; import java.util.ArrayList; import java.util.List; -import com.hy.java.uct.util.EntityRelation; -import com.hy.java.uct.util.UMLClass; +import com.hy.java.uct.util.cd.EntityRelation; +import com.hy.java.uct.util.cd.UMLClass; import com.hy.java.utility.common.Pair; public class Entity { diff --git a/src/main/java/com/hy/java/uct/cdtocode/util/MappedFile.java b/src/main/java/com/hy/java/uct/cdtocode/util/MappedFile.java index 526b0932c827b61e2cbdb9bc615e80ecd7cccf2b..43f0d8ce204ef21100349309a16eebeb2575bfe4 100644 --- a/src/main/java/com/hy/java/uct/cdtocode/util/MappedFile.java +++ b/src/main/java/com/hy/java/uct/cdtocode/util/MappedFile.java @@ -3,7 +3,7 @@ package com.hy.java.uct.cdtocode.util; import java.util.ArrayList; import java.util.List; -import com.hy.java.uct.util.EntityRelation; +import com.hy.java.uct.util.cd.EntityRelation; import com.hy.java.utility.common.Pair; /** diff --git a/src/main/java/com/hy/java/uct/cdtocode/vsmlsi/SimNameTracer.java b/src/main/java/com/hy/java/uct/cdtocode/vsmlsi/SimNameTracer.java index 3d768b1fa61a0b1c00288578c1b6696a12f22ef6..d4ea79a55e295e87aba854d6d09174920aa35feb 100644 --- a/src/main/java/com/hy/java/uct/cdtocode/vsmlsi/SimNameTracer.java +++ b/src/main/java/com/hy/java/uct/cdtocode/vsmlsi/SimNameTracer.java @@ -6,7 +6,7 @@ import java.util.Map; import com.hy.java.uct.cdtocode.reader.CDReader; import com.hy.java.uct.cdtocode.reader.CodeReader; -import com.hy.java.uct.util.UMLClass; +import com.hy.java.uct.util.cd.UMLClass; public class SimNameTracer { /** diff --git a/src/main/java/com/hy/java/uct/cdtocode/vsmlsi/VSMAndLSIDataGenerator.java b/src/main/java/com/hy/java/uct/cdtocode/vsmlsi/VSMAndLSIDataGenerator.java index 00d5190b0cfeec241088a25f9fc4e17545970d0d..9108935fc1c67fa83d0aa3b349a2802b9281793a 100644 --- a/src/main/java/com/hy/java/uct/cdtocode/vsmlsi/VSMAndLSIDataGenerator.java +++ b/src/main/java/com/hy/java/uct/cdtocode/vsmlsi/VSMAndLSIDataGenerator.java @@ -7,7 +7,7 @@ import java.util.Set; import com.hy.java.uct.cdtocode.reader.CDReader; import com.hy.java.uct.cdtocode.reader.CodeReader; -import com.hy.java.uct.util.UMLClass; +import com.hy.java.uct.util.cd.UMLClass; import com.hy.java.utility.common.FileEditor; import jxl.Sheet; diff --git a/src/main/java/com/hy/java/uct/cdtocode/vsmlsi/VSMAndLSIParser.java b/src/main/java/com/hy/java/uct/cdtocode/vsmlsi/VSMAndLSIParser.java index 6a2af08a0a388ac21f05fc523b10077c4c236578..7f78f009152e3656dc67b60ad5a31519a4ef9310 100644 --- a/src/main/java/com/hy/java/uct/cdtocode/vsmlsi/VSMAndLSIParser.java +++ b/src/main/java/com/hy/java/uct/cdtocode/vsmlsi/VSMAndLSIParser.java @@ -6,7 +6,7 @@ import java.util.Map; import java.util.Set; import com.hy.java.uct.cdtocode.reader.CDReader; -import com.hy.java.uct.util.UMLClass; +import com.hy.java.uct.util.cd.UMLClass; import com.hy.java.utility.common.FileEditor; import jxl.Sheet; diff --git a/src/main/java/com/hy/java/uct/sdtocode/SDToCodeTracer.java b/src/main/java/com/hy/java/uct/sdtocode/SDToCodeTracer.java index 28b62f258238e8bcaf349d07781e12c1ae5fe281..0d9b20bf4ddf38a1921f760fd51609457def763f 100644 --- a/src/main/java/com/hy/java/uct/sdtocode/SDToCodeTracer.java +++ b/src/main/java/com/hy/java/uct/sdtocode/SDToCodeTracer.java @@ -1,5 +1,22 @@ package com.hy.java.uct.sdtocode; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import com.hy.java.uct.sdtocode.mapper.CodeMessageMapper; +import com.hy.java.uct.sdtocode.mapper.DocAnalyzer; +import com.hy.java.uct.sdtocode.reader.CodeReader; +import com.hy.java.uct.sdtocode.reader.DocReader; +import com.hy.java.uct.sdtocode.reader.SDReader; +import com.hy.java.uct.util.sd.UMLObject; + +/** + * 顺序图的追踪分为两类：1、以对象为主。2、以消息为主。最后设计一种方法综合两类结果。 + * + * 消息必须按顺序记录，且需将return配对。return中的数据类型可以为正向消息的追踪提供参考。 + * + */ public class SDToCodeTracer { /** * 将要追踪的顺序图放在sd_dir目录下 @@ -7,7 +24,7 @@ public class SDToCodeTracer { private static final String sd_dir = System.getProperty("user.dir") + "\\src\\main\\resources\\sdtocode\\sd\\"; /** - * 将设计文档放在doc_dir目录下 + * 将类图追踪结果放在doc_dir目录下 */ private static final String doc_dir = System.getProperty("user.dir") + "\\src\\main\\resources\\sdtocode\\doc\\"; @@ -25,7 +42,98 @@ public class SDToCodeTracer { * 针对顺序图及其相关文档，追踪到代码中的类 */ public static void main(String[] args) { - // TODO Auto-generated method stub - + /* + * 1、读取模型信息 + */ + // 读取完UML图识别结果后，将实体信息保存在classes_in_CD里。形式为 + /* + * Hadoop HDFS1 + */ + Map objs_in_SD = SDReader.read(sd_dir + "sd-Hadoop HDFS1.txt"); + /* + * Hadoop MapReduce + */ + // Map classes_in_CD = CDReader.read(cd_dir + "cd-Hadoop MapReduce.txt"); + // 检查结果，可注释掉 + SDReader.check(objs_in_SD); + /* + * 2、读取文档信息 + * + * 做“自己方法内有无文档的对比”的实验时，在不导入任何文档即可 + */ + List doc_dir_ls = new ArrayList<>(); + // 在这儿添加多个文件 + /* + * Hadoop HDFS + */ + doc_dir_ls.add(doc_dir + "Hadoop HDFS\\Hadoop architectural overview.txt"); + doc_dir_ls.add(doc_dir + "Hadoop HDFS\\Hadoop clusters with Kove® XPD™ persistent memory.txt"); + doc_dir_ls.add(doc_dir + "Hadoop HDFS\\HADOOP DISTRIBUTED FILE SYSTEM (HDFS) ARCHITECTURAL DOCUMENTATION - MODULE VIEW.txt"); + doc_dir_ls.add(doc_dir + "Hadoop HDFS\\Hadoop Distributed File System (HDFS) Architecture – A Guide to HDFS for Every Data Engineer.txt"); + doc_dir_ls.add(doc_dir + "Hadoop HDFS\\HADOOP ECOSYSTEM.txt"); + doc_dir_ls.add(doc_dir + "Hadoop HDFS\\Hadoop HDFS Architecture Explanation and Assumptions.txt"); + doc_dir_ls.add(doc_dir + "Hadoop HDFS\\HDFS Architecture Guide.txt"); + doc_dir_ls.add(doc_dir + "Hadoop HDFS\\HDFS Architecture.txt"); + doc_dir_ls.add(doc_dir + "Hadoop HDFS\\HDFS.txt"); + doc_dir_ls.add(doc_dir + "Hadoop HDFS\\Key Design of HDFS Architecture.txt"); + doc_dir_ls.add(doc_dir + "Hadoop HDFS\\The Hadoop Distributed File System Architecture and Design.txt"); + doc_dir_ls.add(doc_dir + "Hadoop HDFS\\Towards A Scalable HDFS Architecture.txt"); + /* + * Hadoop MapReduce + */ + /* + * doc_dir_ls.add(doc_dir + "Hadoop MapReduce\\A BIG DATA PROCESSING FRAMEWORK BASED ON MAPREDUCE WITH APPLICATION TO INTERNET OF THINGS.txt"); doc_dir_ls.add(doc_dir + + * "Hadoop MapReduce\\Apache Hadoop Architecture – HDFS, YARN & MapReduce.txt"); doc_dir_ls.add(doc_dir + "Hadoop MapReduce\\Big Data Analysis Challenges and Solutions.txt"); doc_dir_ls.add(doc_dir + + * "Hadoop MapReduce\\Big Data Management on Wireless Sensor Networks.txt"); doc_dir_ls.add(doc_dir + "Hadoop MapReduce\\Hadoop - MapReduce.txt"); doc_dir_ls.add(doc_dir + + * "Hadoop MapReduce\\Hadoop Architecture in Detail – HDFS, Yarn & MapReduce.txt"); doc_dir_ls.add(doc_dir + "Hadoop MapReduce\\Hadoop MapReduce- Java-based Processing Framework for Big Data.txt"); + * doc_dir_ls.add(doc_dir + "Hadoop MapReduce\\MapReduce – Components.txt"); doc_dir_ls.add(doc_dir + "Hadoop MapReduce\\MapReduce Architecture1.txt"); doc_dir_ls.add(doc_dir + + * "Hadoop MapReduce\\MapReduce Architecture2.txt"); doc_dir_ls.add(doc_dir + "Hadoop MapReduce\\MapReduce Architecture3.txt"); doc_dir_ls.add(doc_dir + "Hadoop MapReduce\\MapReduce Tutorial.txt"); + * doc_dir_ls.add(doc_dir + "Hadoop MapReduce\\MapReduce Working and Components.txt"); doc_dir_ls.add(doc_dir + "Hadoop MapReduce\\MapReduce.txt"); doc_dir_ls.add(doc_dir + + * "Hadoop MapReduce\\mapreduce_hadoop2.txt"); doc_dir_ls.add(doc_dir + "Hadoop MapReduce\\Understanding MapReduce in Hadoop.txt"); doc_dir_ls.add(doc_dir + + * "Hadoop MapReduce\\What are the components of MapReduce.txt"); doc_dir_ls.add(doc_dir + "Hadoop MapReduce\\What Is MapReduce Architecture An Important Overview For 2021.txt"); + */ + // 实际使用的Map，保存每份文档地址及其内容 + Map> dir_sentences_map = DocReader.readDocs(doc_dir_ls); + /* + * 3、读取code path指定的目录下所有java文件 + * + * <类全称（包+类名）, java_file_path> + */ + // 记得改这里的路径 + // Map classFullName_javaFileDir_map = CodeReader.read(code_dir + "code path-hdfs"); + Map classFullName_javaFileDir_map = null; + // Map classFullName_javaFileDir_map = CodeReader.read(code_dir + "code path-mr"); + // 检查结果，可注释掉 + // CodeReader.check(classFullName_javaFileDir_map); + /* + * 4、分析文档信息。实际相当于增加类图中的UMLclass、类本身的内容、类之间关系 + * + * 核心思想：我们的目标是以模型为准，将模型元素追踪到代码。由于模型元素抽象层次较高，所以直接追踪未必追踪的到，因此需找与模型元素具有关系的其他元素（向下一层层找），将模型元素降维、直至能与代码元素对应上为止。 + * + * 以下追踪实际包含3层：图中的类、文档中的实体、代码中的类。 + * + * 先将图中的类追踪到包含文档信息的实体（一对多），再将这些实体追踪到代码中的类（也是一对多），所以是“一→多→更多”。 + * + * 其中，实体间还会根据语义分析得到一些关系 + */ + Map classShortName_classObj_mappedByDoc = DocAnalyzer.analyze(objs_in_SD, dir_sentences_map); + // 检查结果，可注释掉 + // DocAnalyzer.check(classShortName_classObj_mappedByDoc); + /* + * 5、遍历模型中的实体元素（类或对象），针对每个元素，在code中寻找能匹配的java文件 + * + * 基于启发式模糊匹配的UML类图与代码追踪方法：首先针对类图中的类和代码中的类，基于类的名称进行字符串完全匹配，从而建立确定的初始追踪；基于同义词、 词缀词典等语料库，定义基于类名称匹配和关联关系的启发式匹配规则，研究基于模糊匹配技术的追踪关系建立方法，基于初始追踪和启发式规则， 对类名不一致的模型和代码元素进行启发式追踪，扩展初始追踪关系。 + */ + Map mapped_classes = CodeMessageMapper.map(classShortName_classObj_mappedByDoc, classFullName_javaFileDir_map, false); + /* + * Hadoop HDFS + */ + CodeMessageMapper.save(mapped_classes, res_dir + "Hadoop HDFS-noadd.xls"); + /* + * Hadoop MapReduce + */ + // CodeRelationMapper.save(mapped_classes, res_dir + "Hadoop MapReduce-noadd.xls"); + // 检查结果，可注释掉 + // CodeRelationMapper.check(res_dir + "Apache OODT File Manager.xls"); } } diff --git a/src/main/java/com/hy/java/uct/sdtocode/mapper/CodeMessageMapper.java b/src/main/java/com/hy/java/uct/sdtocode/mapper/CodeMessageMapper.java new file mode 100644 index 0000000000000000000000000000000000000000..fdf9d601669eb5d1eb1363bcfc24050ef8e4ec3c --- /dev/null +++ b/src/main/java/com/hy/java/uct/sdtocode/mapper/CodeMessageMapper.java @@ -0,0 +1,19 @@ +package com.hy.java.uct.sdtocode.mapper; + +import java.util.Map; + +import com.hy.java.uct.util.sd.UMLObject; + +public class CodeMessageMapper { + + public static Map map(Map classShortName_classObj_mappedByDoc, Map classFullName_javaFileDir_map, boolean b) { + // TODO Auto-generated method stub + return null; + } + + public static void save(Map mapped_classes, String string) { + // TODO Auto-generated method stub + + } + +} diff --git a/src/main/java/com/hy/java/uct/sdtocode/mapper/DocAnalyzer.java b/src/main/java/com/hy/java/uct/sdtocode/mapper/DocAnalyzer.java new file mode 100644 index 0000000000000000000000000000000000000000..37b8b0fbef0976fc9f1fb15eb97cac2bb92bafdb --- /dev/null +++ b/src/main/java/com/hy/java/uct/sdtocode/mapper/DocAnalyzer.java @@ -0,0 +1,15 @@ +package com.hy.java.uct.sdtocode.mapper; + +import java.util.List; +import java.util.Map; + +import com.hy.java.uct.util.sd.UMLObject; + +public class DocAnalyzer { + + public static Map analyze(Map objs_in_SD, Map> dir_sentences_map) { + // TODO Auto-generated method stub + return null; + } + +} diff --git a/src/main/java/com/hy/java/uct/sdtocode/reader/CodeReader.java b/src/main/java/com/hy/java/uct/sdtocode/reader/CodeReader.java new file mode 100644 index 0000000000000000000000000000000000000000..05f5465fcf00395e8c717a3368ded8c4ccf4736a --- /dev/null +++ b/src/main/java/com/hy/java/uct/sdtocode/reader/CodeReader.java @@ -0,0 +1,91 @@ +package com.hy.java.uct.sdtocode.reader; + +import java.io.File; +import java.io.FileNotFoundException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import com.github.javaparser.StaticJavaParser; +import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.PackageDeclaration; +import com.hy.java.utility.common.FileEditor; +import com.hy.java.utility.common.Traverser; +import com.hy.java.utility.common.Traverser.FileNode; + +public class CodeReader { + + public static Map read(String code_path_file_path) { + // 该Map保存所有遍历到的java文件，形式为 + Map result = new HashMap<>(); + // 读取配置文件 + FileEditor code_path_file = new FileEditor(code_path_file_path); + // 配置文件指定的代码根目录所对应的FileNode。遍历其children下的java文件即可 + FileNode code_files_root = Traverser.traverseDir(code_path_file.readFileToString()); + // 遍历 + System.out.println("正在遍历" + code_files_root.path + "下的源码文件，请稍候..."); + filterJavaFileFromFN(code_files_root, result); + System.out.println("已遍历完java文件"); + return result; + } + + /** + * 递归遍历code_files_root下的所有java文件，以存在Map中 + */ + private static void filterJavaFileFromFN(FileNode code_files_root, Map java_files) { + if (code_files_root.children == null) { + if (code_files_root.path.endsWith(".java")) { + // 过滤\src\test\下的文件 + if (!code_files_root.path.contains("\\src\\test\\")) { + // 这里put的key是类全称（包+类名） + try { + CompilationUnit cu = StaticJavaParser.parse(new File(code_files_root.path)); + Optional package_declaration = cu.getPackageDeclaration(); + if (!package_declaration.isEmpty()) { + java_files.put(package_declaration.get().getNameAsString() + "." + code_files_root.path.substring(code_files_root.path.lastIndexOf("\\") + 1, code_files_root.path.lastIndexOf(".")), code_files_root.path); + } + } catch (FileNotFoundException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + } else { + for (FileNode code_file_or_dir : code_files_root.children) { + filterJavaFileFromFN(code_file_or_dir, java_files); + } + } + } + + /** + * 检查一下是不是遍历了所有java文件 + */ + public static void check(Map java_files) { + Set class_names = java_files.keySet(); + for (String class_name : class_names) { + System.out.println(class_name + "\t" + java_files.get(class_name)); + } + } + + /** + * 根据API文档，如果有包名与图中类名相似，则记录该包下所有的class + * + * 这是一次性方法，用完就注释掉 + */ + public static Set findPcks(String code_path_file_path) { + Set res = new HashSet<>(); + Map classFullName_javaFileDir_map = CodeReader.read(code_path_file_path); + Set clsFullNames = classFullName_javaFileDir_map.keySet(); + for (String clsFullName : clsFullNames) { + res.add(getParentPck(clsFullName)); + } + return res; + } + + private static String getParentPck(String clsFullName) { + String temp = clsFullName.substring(0, clsFullName.lastIndexOf(".")); + return temp.substring(temp.lastIndexOf(".") + 1); + } +} diff --git a/src/main/java/com/hy/java/uct/sdtocode/reader/DocReader.java b/src/main/java/com/hy/java/uct/sdtocode/reader/DocReader.java new file mode 100644 index 0000000000000000000000000000000000000000..0664369faf89bcba076a19ce680667ededccc72a --- /dev/null +++ b/src/main/java/com/hy/java/uct/sdtocode/reader/DocReader.java @@ -0,0 +1,21 @@ +package com.hy.java.uct.sdtocode.reader; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.hy.java.utility.common.FileEditor; + +public class DocReader { + /** + * 读取文档信息 + */ + public static Map> readDocs(List doc_ls) { + Map> res = new HashMap<>(); + for (String doc_dir : doc_ls) { + FileEditor doc_file = new FileEditor(doc_dir); + res.put(doc_dir, doc_file.readLines()); + } + return res; + } +} diff --git a/src/main/java/com/hy/java/uct/sdtocode/reader/SDReader.java b/src/main/java/com/hy/java/uct/sdtocode/reader/SDReader.java new file mode 100644 index 0000000000000000000000000000000000000000..1f94a181b0c8a999ef0ba7bafdaf4645a7dcb35d --- /dev/null +++ b/src/main/java/com/hy/java/uct/sdtocode/reader/SDReader.java @@ -0,0 +1,69 @@ +package com.hy.java.uct.sdtocode.reader; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import com.hy.java.uct.util.sd.Message; +import com.hy.java.uct.util.sd.UMLObject; +import com.hy.java.utility.common.FileEditor; + +public class SDReader { + /** + * 读取UML图识别结果，将实体信息保存在result里并返回 + */ + public static Map read(String sd_recog_res_path) { + Map result = new HashMap<>(); + // UML图识别结果 + FileEditor model_file = new FileEditor(sd_recog_res_path); + // 保存对象名。对象之间以“#”隔开 + String[] obj_strs = model_file.readFileToString().split("#"); + for (String obj_str : obj_strs) { + UMLObject UML_obj = new UMLObject(); + // 消息用“@”隔开 + String[] obj_info_strs = obj_str.split("@"); + // 对象名 + UML_obj.setTitle(obj_info_strs[0].trim()); + result.put(UML_obj.getTitle(), UML_obj); + } + // 根据对象名保存关系（出） + for (String obj_str : obj_strs) { + String[] obj_info_strs = obj_str.split("@"); + UMLObject UML_obj = result.get(obj_info_strs[0].trim()); + // 出 + if (obj_info_strs.length >= 2) { + if (!obj_info_strs[1].isBlank()) { + // 关系用“￥”隔开 + String[] out_msgs = obj_info_strs[1].split("￥"); + for (String out_msg_str : out_msgs) { + Message msg = new Message(); + String[] msg_info = out_msg_str.split("%"); + if (msg_info.length >= 2) { + msg.source = result.get(msg_info[0].trim()); + msg.target = result.get(msg_info[1].trim()); + if (msg_info.length == 3) { + msg.msg = msg_info[2]; + } else { + msg.msg = ""; + } + UML_obj.out_msgs.add(msg); + } + } + } + } + } + return result; + } + + public static void check(Map objs_in_SD) { + Set keys = objs_in_SD.keySet(); + for (String key : keys) { + UMLObject uo = objs_in_SD.get(key); + System.out.println("对象名：" + uo.getTitle()); + for (Message m : uo.out_msgs) { + System.out.println("出关：" + m.source.getTitle() + "→" + m.target.getTitle() + "说" + m.msg); + } + System.out.println("======================="); + } + } +} diff --git a/src/main/java/com/hy/java/uct/umlrecog/ClassDiagramRecognizer.java b/src/main/java/com/hy/java/uct/umlrecog/ClassDiagramRecognizer.java index 56a245870fdd15d4aa9f48d9f64508bc070f0ba5..99f4ea845a332cf7e119b63f5c750c8ed5629758 100644 --- a/src/main/java/com/hy/java/uct/umlrecog/ClassDiagramRecognizer.java +++ b/src/main/java/com/hy/java/uct/umlrecog/ClassDiagramRecognizer.java @@ -10,8 +10,8 @@ import org.opencv.core.Mat; import com.hy.java.uct.umlrecog.cddetector.ClassDetector; import com.hy.java.uct.umlrecog.cddetector.ClassRelationDetector; -import com.hy.java.uct.util.ImgRelation; -import com.hy.java.uct.util.UMLClass; +import com.hy.java.uct.util.cd.ImgRelation; +import com.hy.java.uct.util.cd.UMLClass; import com.hy.java.utility.common.FileEditor; import com.hy.java.utility.common.Pair; import com.hy.java.utility.common.Traverser; diff --git a/src/main/java/com/hy/java/uct/umlrecog/SequenceDiagramRecognizer.java b/src/main/java/com/hy/java/uct/umlrecog/SequenceDiagramRecognizer.java index 8196505c392b27e60b70e674adfa9efeb55d9d30..959e49363489d2acfb9e2a1e080c04b04165f5e6 100644 --- a/src/main/java/com/hy/java/uct/umlrecog/SequenceDiagramRecognizer.java +++ b/src/main/java/com/hy/java/uct/umlrecog/SequenceDiagramRecognizer.java @@ -7,8 +7,8 @@ import org.opencv.core.Mat; import com.hy.java.uct.umlrecog.sddetector.MessageDetector; import com.hy.java.uct.umlrecog.sddetector.ObjectDetector; -import com.hy.java.uct.umlrecog.util.Message; -import com.hy.java.uct.umlrecog.util.UMLObject; +import com.hy.java.uct.util.sd.Message; +import com.hy.java.uct.util.sd.UMLObject; import com.hy.java.utility.common.FileEditor; import com.hy.java.utility.common.Pair; import com.hy.java.utility.common.Traverser; diff --git a/src/main/java/com/hy/java/uct/umlrecog/cddetector/ClassDetector.java b/src/main/java/com/hy/java/uct/umlrecog/cddetector/ClassDetector.java index 47905a2cd1b3a73841c148af8a031aa575c7ab35..98a75b5ee1df4fdfe38a34b2e269ab72642e894f 100644 --- a/src/main/java/com/hy/java/uct/umlrecog/cddetector/ClassDetector.java +++ b/src/main/java/com/hy/java/uct/umlrecog/cddetector/ClassDetector.java @@ -15,7 +15,7 @@ import org.opencv.imgproc.Imgproc; import com.hy.java.uct.umlrecog.UMLDiagramRecognizer; import com.hy.java.uct.umlrecog.util.ImgProcessor; import com.hy.java.uct.umlrecog.util.Rectangle; -import com.hy.java.uct.util.UMLClass; +import com.hy.java.uct.util.cd.UMLClass; import com.hy.java.utility.common.Pair; import net.sourceforge.tess4j.ITesseract; diff --git a/src/main/java/com/hy/java/uct/umlrecog/cddetector/ClassRelationDetector.java b/src/main/java/com/hy/java/uct/umlrecog/cddetector/ClassRelationDetector.java index 051a6c8e235081f65778a5b33cb5de286aaa79be..3b19a9b46923edcd28b6f7236a098e2ba8228256 100644 --- a/src/main/java/com/hy/java/uct/umlrecog/cddetector/ClassRelationDetector.java +++ b/src/main/java/com/hy/java/uct/umlrecog/cddetector/ClassRelationDetector.java @@ -19,8 +19,8 @@ import org.opencv.imgproc.Imgproc; import com.hy.java.uct.umlrecog.util.ImgProcessor; import com.hy.java.uct.umlrecog.util.Line; import com.hy.java.uct.umlrecog.util.PolygonalLine; -import com.hy.java.uct.util.ImgRelation; -import com.hy.java.uct.util.UMLClass; +import com.hy.java.uct.util.cd.ImgRelation; +import com.hy.java.uct.util.cd.UMLClass; import com.hy.java.utility.common.Pair; public class ClassRelationDetector { diff --git a/src/main/java/com/hy/java/uct/umlrecog/sddetector/MessageDetector.java b/src/main/java/com/hy/java/uct/umlrecog/sddetector/MessageDetector.java index ca2ba5264d1c7bcc716e38eacf8f6036c4e0945e..823ddd286a271df59922c8489012715a20beeb1b 100644 --- a/src/main/java/com/hy/java/uct/umlrecog/sddetector/MessageDetector.java +++ b/src/main/java/com/hy/java/uct/umlrecog/sddetector/MessageDetector.java @@ -17,8 +17,8 @@ import org.opencv.imgproc.Imgproc; import com.hy.java.uct.umlrecog.UMLDiagramRecognizer; import com.hy.java.uct.umlrecog.util.ImgProcessor; import com.hy.java.uct.umlrecog.util.Line; -import com.hy.java.uct.umlrecog.util.Message; -import com.hy.java.uct.umlrecog.util.UMLObject; +import com.hy.java.uct.util.sd.Message; +import com.hy.java.uct.util.sd.UMLObject; import com.hy.java.utility.common.Pair; import net.sourceforge.tess4j.ITesseract; diff --git a/src/main/java/com/hy/java/uct/umlrecog/sddetector/ObjectDetector.java b/src/main/java/com/hy/java/uct/umlrecog/sddetector/ObjectDetector.java index 0751e8ac64cc3de89d73d2c537d330e80fcb5652..27a8c3df4da6317b5c26cf8723da48779e2753cf 100644 --- a/src/main/java/com/hy/java/uct/umlrecog/sddetector/ObjectDetector.java +++ b/src/main/java/com/hy/java/uct/umlrecog/sddetector/ObjectDetector.java @@ -16,7 +16,7 @@ import org.opencv.ml.EM; import com.hy.java.uct.umlrecog.UMLDiagramRecognizer; import com.hy.java.uct.umlrecog.util.ImgProcessor; import com.hy.java.uct.umlrecog.util.Rectangle; -import com.hy.java.uct.umlrecog.util.UMLObject; +import com.hy.java.uct.util.sd.UMLObject; import com.hy.java.utility.common.Pair; import net.sourceforge.tess4j.ITesseract; diff --git a/src/main/java/com/hy/java/uct/util/EntityRelation.java b/src/main/java/com/hy/java/uct/util/cd/EntityRelation.java similarity index 91% rename from src/main/java/com/hy/java/uct/util/EntityRelation.java rename to src/main/java/com/hy/java/uct/util/cd/EntityRelation.java index 70f11f0e4a4a56e132ee527976e568ede62a3c89..6e872ace3735d4fdcba20494f3809e7360028ef8 100644 --- a/src/main/java/com/hy/java/uct/util/EntityRelation.java +++ b/src/main/java/com/hy/java/uct/util/cd/EntityRelation.java @@ -1,4 +1,4 @@ -package com.hy.java.uct.util; +package com.hy.java.uct.util.cd; import com.hy.java.uct.cdtocode.util.Entity; diff --git a/src/main/java/com/hy/java/uct/util/ImgRelation.java b/src/main/java/com/hy/java/uct/util/cd/ImgRelation.java similarity index 91% rename from src/main/java/com/hy/java/uct/util/ImgRelation.java rename to src/main/java/com/hy/java/uct/util/cd/ImgRelation.java index d8cd2286a71ab933398f3f09050e0dcddc0e4469..544870a9320adfc3075efdb58d9e9723eeff369d 100644 --- a/src/main/java/com/hy/java/uct/util/ImgRelation.java +++ b/src/main/java/com/hy/java/uct/util/cd/ImgRelation.java @@ -1,4 +1,4 @@ -package com.hy.java.uct.util; +package com.hy.java.uct.util.cd; import com.hy.java.uct.umlrecog.util.PolygonalLine; diff --git a/src/main/java/com/hy/java/uct/util/UMLClass.java b/src/main/java/com/hy/java/uct/util/cd/UMLClass.java similarity index 94% rename from src/main/java/com/hy/java/uct/util/UMLClass.java rename to src/main/java/com/hy/java/uct/util/cd/UMLClass.java index 3426345f7efba8e4ff0e5c027112e39e956a363b..2963cb099a4bcfa39379dcc4481170b5af8176a4 100644 --- a/src/main/java/com/hy/java/uct/util/UMLClass.java +++ b/src/main/java/com/hy/java/uct/util/cd/UMLClass.java @@ -1,4 +1,4 @@ -package com.hy.java.uct.util; +package com.hy.java.uct.util.cd; import java.util.ArrayList; import java.util.HashSet; diff --git a/src/main/java/com/hy/java/uct/umlrecog/util/Message.java b/src/main/java/com/hy/java/uct/util/sd/Message.java similarity index 63% rename from src/main/java/com/hy/java/uct/umlrecog/util/Message.java rename to src/main/java/com/hy/java/uct/util/sd/Message.java index 90865620f7bb55a1fad7b0b782725ced87244724..250880e3fea4fd1a939f036d8c5c76d744f09d40 100644 --- a/src/main/java/com/hy/java/uct/umlrecog/util/Message.java +++ b/src/main/java/com/hy/java/uct/util/sd/Message.java @@ -1,4 +1,7 @@ -package com.hy.java.uct.umlrecog.util; +package com.hy.java.uct.util.sd; + +import com.hy.java.uct.umlrecog.util.Line; +import com.hy.java.uct.umlrecog.util.PolygonalLine; public class Message { @@ -18,4 +21,7 @@ public class Message { public Message(Line line) { this.line = line; } + + public Message() { + } } diff --git a/src/main/java/com/hy/java/uct/umlrecog/util/UMLObject.java b/src/main/java/com/hy/java/uct/util/sd/UMLObject.java similarity index 80% rename from src/main/java/com/hy/java/uct/umlrecog/util/UMLObject.java rename to src/main/java/com/hy/java/uct/util/sd/UMLObject.java index 530ac65ac514a5ea5d7f859d91da4004a004b2fb..075db36c51885bb4864a32847bb6df721de34314 100644 --- a/src/main/java/com/hy/java/uct/umlrecog/util/UMLObject.java +++ b/src/main/java/com/hy/java/uct/util/sd/UMLObject.java @@ -1,8 +1,10 @@ -package com.hy.java.uct.umlrecog.util; +package com.hy.java.uct.util.sd; import java.util.ArrayList; import java.util.List; +import com.hy.java.uct.umlrecog.util.Rectangle; + public class UMLObject { /** * 只在mergeIntoObject时使用 diff --git a/src/main/resources/sdtocode/code/code path-fm b/src/main/resources/sdtocode/code/code path-fm deleted file mode 100644 index 31118504bd339b5a3d8c18b9c3fe0e2d76ad84e2..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/code/code path-fm +++ /dev/null @@ -1 +0,0 @@ -D:\eclipse-committers\Apache OODT File Manager \ No newline at end of file diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/A FRAMEWORK FOR COLLABORATIVE REVIEW OF CANDIDATE EVENTS IN HIGH DATA RATE STREAMS THE V-FASTR EXPERIMENT AS A CASE STUDY-relation.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/A FRAMEWORK FOR COLLABORATIVE REVIEW OF CANDIDATE EVENTS IN HIGH DATA RATE STREAMS THE V-FASTR EXPERIMENT AS A CASE STUDY-relation.txt deleted file mode 100644 index cac5cd6101b4bec2c7118f647b883acbc68e0b00..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/doc/Apache OODT File Manager/A FRAMEWORK FOR COLLABORATIVE REVIEW OF CANDIDATE EVENTS IN HIGH DATA RATE STREAMS THE V-FASTR EXPERIMENT AS A CASE STUDY-relation.txt +++ /dev/null @@ -1,742 +0,0 @@ -v-fastr experiment&a case study andrew f. hart and Pasadena and CA 91109 and USA&依赖 -FRAMEWORK&radio-frequency energy&依赖 -California Institute&Technology&AGGREGATION -v-fastr experiment&a case study andrew f. hart and Pasadena and CA 91109 and USA&依赖 -FRAMEWORK&bright millisecond pulse&依赖 -bright millisecond pulse&radio-frequency energy&AGGREGATION -shortduration pulse&known object&依赖 -shortduration pulse&pulsar&依赖 -identification and verification&event&AGGREGATION -one major goal&very long baseline array ( vlba ) fast transient experiment ( v-fastr )&AGGREGATION -V-FASTR&“ commensal ” ( piggy-back ) approach&依赖 -Raw datum&buffer memory&依赖 -buffer memory&comprehensive off-line analysis&依赖 -astrophysical origin&detection&AGGREGATION -promising signal&more in-depth investigation&依赖 -candidate&analyst&依赖 -we&metadata-driven , collaborative candidate review framework&依赖 -timely analysis&fast transient detection candidate&AGGREGATION -software pipeline&pipeline&GENERALIZATION -framework&software pipeline&依赖 -framework&metadata processing&依赖 -facilitates browsing and inspection&available metada&AGGREGATION -current primary goal&radio astronomy&AGGREGATION -INTRODUCTION One&” ( and cordes et al. 2004 )&依赖 -INTRODUCTION One&cordes et al. 2004 )&依赖 -INTRODUCTION One&cordes et al. 2004 )&依赖 -INTRODUCTION One&” ( and cordes et al. 2004 )&依赖 -INTRODUCTION One&” ( and cordes et al. 2004 )&依赖 -INTRODUCTION One&cordes et al. 2004 )&依赖 -INTRODUCTION One¤t primary goal&AGGREGATION -scientific thrust&contrast&依赖 -scientific thrust&known source&依赖 -scientific thrust&transient event&依赖 -generate catalog&known source&AGGREGATION -scientific thrust&generate catalog&依赖 -scope and distribution&different transient source&AGGREGATION -We&different transient source&依赖 -We&scope and distribution&依赖 -“ most exciting&sources ”&依赖 -new class&sources ”&AGGREGATION -“ most exciting&original )&依赖 -“ most exciting&sources ”&依赖 -discovery&sources ”&AGGREGATION -“ most exciting&new class&依赖 -discovery&new class&AGGREGATION -“ most exciting&( italics&依赖 -Radio telescope&sky&依赖 -their&data& -detection and characterization and ” which&“ fast radio transient&AGGREGATION -detection and characterization and ” which¤t particular interest&依赖 -detection and characterization and ” which¤t particular interest&依赖 -small fraction&second&AGGREGATION -V-FASTR experiment&radio astronomy experiment&依赖 -one&new breed&AGGREGATION -new breed&radio astronomy experiment&AGGREGATION -V-FASTR experiment&transient radio signal&依赖 -V-FASTR experiment&experiment&GENERALIZATION -V-FASTR experiment&transient radio signal&依赖 -V-FASTR experiment&radio astronomy experiment&依赖 -its&instrument& -regular processing activity&host instrument&AGGREGATION -V-FASTR experiment&2 and 10 telescope&依赖 -V-FASTR experiment&National Radio Astronomy Observatory ʼs&依赖 -V-FASTR experiment&National Radio Astronomy Observatory ʼs&依赖 -2 and 10 telescope&National Radio Astronomy Observatory ʼs ( nrao ) very long baseline array ( vlba ) ( romney 2010 )&AGGREGATION -V-FASTR experiment&2 and 10 telescope&依赖 -V-FASTR experiment&( rfus ) and potentially genuine astronomical pulse&依赖 -V-FASTR experiment&configuration&依赖 -VLBA&10 25 m telescope&依赖 -V-FASTR experiment&thompson et al. 2011 )&依赖 -instance&terrestrial Radio Frequency Interference ( rfus ) and potentially genuine astronomical pulse&AGGREGATION -huge volume&entire observing session infeasible&依赖 -huge volume&full record&依赖 -huge volume&full record&依赖 -its&operation& -course&operation&AGGREGATION -full record&entire observing session infeasible&AGGREGATION -huge volume&entire observing session infeasible&依赖 -huge volume&entire observing session infeasible&依赖 -huge volume&raw time-series voltage datum&AGGREGATION -huge volume&full record&依赖 -considerable effort&noisy and often incomplete datum&依赖 -considerable effort&consequence&依赖 -considerable effort&interesting signal&依赖 -considerable effort&fine-tuning algorithm&依赖 -considerable effort&realtime identification&依赖 -realtime identification&interesting signal&AGGREGATION -considerable effort&Thompson et al. 2011&依赖 -datum&distributed V-FASTR science team&依赖 -member&distributed V-FASTR science team&AGGREGATION -datum&daily basis&依赖 -datum&member&依赖 -V-FASTR science team&science team&GENERALIZATION -practical workload constraint&candidate detection&依赖 -V-FASTR experiment&several important resource constraint&依赖 -practical workload constraint&candidate detection&依赖 -we&V-FASTR experiment&依赖 -automatic capture and organization&metada&AGGREGATION -framework&set&依赖 -set&software component&AGGREGATION -framework&candidate event&依赖 -framework&software component&依赖 -collaborative perusal and inspection&related imagery datum&AGGREGATION -rest&paper&AGGREGATION -rest&system&依赖 -rest&system&依赖 -we&more general context&依赖 -we&Section 2&依赖 -our&project& -we&project&依赖 -architectural description&system&AGGREGATION -Section 3&methodology&依赖 -evaluation&Astronomical Journal 149:23 ( 7pp )&AGGREGATION -We&evaluation&依赖 -We&Astronomical Journal&依赖 -We&149:23 ( 7pp )&依赖 -our&experience& -development&scientific data system&AGGREGATION -we&VFASTR experiment&依赖 -context&system implementation&AGGREGATION -We&object oriented data technology ( oodt ) project&依赖 -open source information integration platform&framework&依赖 -open source information integration platform¢ral role&依赖 -We&open source information integration platform&依赖 -our&framework& -we&online tool&依赖 -we&several related effort&依赖 -Principal investigator&proposal&依赖 -Principal investigator&VLBA&依赖 -V-FASTR&datum&依赖 -V-FASTR&datum&依赖 -part&routine processing&AGGREGATION -nightly list&candidate&AGGREGATION -V-FASTR&datum&依赖 -candidate list&timely basis&依赖 -candidate list&expert&依赖 -raw datum&significant disk space&依赖 -generation&sky image&AGGREGATION -their&space& -source&signal&AGGREGATION -Software tool&review process&依赖 -Software tool&candidate reviewer&依赖 -review process&process&GENERALIZATION -jpl data management systems and technologies group&software ground data system&依赖 -jpl data management systems and technologies group&software ground data system&依赖 -they&scientist&依赖 -other operational constraint&expected production environment&AGGREGATION -process&project scientist&依赖 -process&close collaboration&依赖 -understanding&processing algorithm&AGGREGATION -sense&scale and throughput requirement&AGGREGATION -broad spectrum&domain&AGGREGATION -group&data system experience&依赖 -group&diverse portfolio&依赖 -group&data system experience&依赖 -group&diverse portfolio&依赖 -group&diverse portfolio&依赖 -group&data system experience&依赖 -group&data system experience&依赖 -group&data system experience&依赖 -group&data system experience&依赖 -group&diverse portfolio&依赖 -group&diverse portfolio&依赖 -group&diverse portfolio&依赖 -diverse portfolio&data system experience&AGGREGATION -Open Source&software component&依赖 -their&design& -suite&software component&AGGREGATION -OODT1&investment&依赖 -long track record&experience&AGGREGATION -mission-specific customization&top&依赖 -realm&scientific data processing system&AGGREGATION -OODT1&individual mission data system&依赖 -OODT One&product&AGGREGATION -reusable platform&configurable component&AGGREGATION -OODT1&return&依赖 -part&NASAʼs Office&AGGREGATION -NASAʼs Office&Space Science&AGGREGATION -product&long track record&AGGREGATION -Open Source&known&依赖 -OODT&architecture&依赖 -component&one another&依赖 -component&XML-RPC2&依赖 -component&standard , open protocol&依赖 -Its&components& -2009 oodt&transition&依赖 -2009 oodt&JPL-internal development project&依赖 -2009 oodt&free and open source software project&依赖 -2009 oodt&JPL-internal development project&依赖 -2009 oodt&free and open source software project&依赖 -2009 oodt&transition&依赖 -OODT&ASF&依赖 -varied group&scientific and commercial endeavor&AGGREGATION -OODT&several public release&依赖 -several OODT component&candidate validation framework&依赖 -several OODT component&core platform&依赖 -core platform&candidate validation framework&AGGREGATION -we&Section 3&依赖 -ready availability&time and budgetary consideration&依赖 -ready availability&project&依赖 -ready availability&time and budgetary consideration&依赖 -ready availability&project&依赖 -ready availability&time and budgetary consideration&依赖 -ready availability&time and budgetary consideration&依赖 -ready availability&OODT component&AGGREGATION -their&pedigree& -ready availability&project&依赖 -ready availability&project&依赖 -several ongoing effort&online tool&依赖 -collaborative review and classification&scientific observation&AGGREGATION -we§ion&依赖 -we&several ongoing effort&依赖 -several ongoing effort&assist&依赖 -part&series&AGGREGATION -Astropulse Astropulse&sky survey&依赖 -Astropulse Astropulse&radio pulse&依赖 -University&berkeley (&AGGREGATION -series&sky survey&AGGREGATION -Astropulse Astropulse&Astropulse&GENERALIZATION -Astropulse Astropulse&series&依赖 -Astropulse project&survey&依赖 -Astropulse project&survey&依赖 -Astropulse project&project&GENERALIZATION -Astropulse project&survey&依赖 -Astropulse project&sky&依赖 -survey&sky&AGGREGATION -Astropulse project&sky&依赖 -Astropulse project&sky&依赖 -v-fastrʼs ability&greater positional accuracy&依赖 -Astropulseʼs use&excellent sensitivity&依赖 -v-fastrʼs ability&greater positional accuracy&依赖 -v-fastrʼs ability&greater positional accuracy&依赖 -v-fastrʼs ability&greater positional accuracy&依赖 -Astropulseʼs use&excellent sensitivity&依赖 -Astropulseʼs use&Areciboʼs enormous single dish telescope&AGGREGATION -variant&SETI@home project&AGGREGATION -Astropulse&same distributed , collaborative volunteer computing infrastructure&依赖 -Astropulse&same distributed , collaborative volunteer computing infrastructure&依赖 -Astropulse&same distributed , collaborative volunteer computing infrastructure&依赖 -Astropulse&same distributed , collaborative volunteer computing infrastructure&依赖 -origin&signal&AGGREGATION -number&intense transformations and calculation&AGGREGATION -intense transformations and calculation&datum&AGGREGATION -unit&computational work&AGGREGATION -use&volunteer computing&AGGREGATION -automated nature&V-FASTRʼs manual review requirement&依赖 -automated nature&approach&AGGREGATION -automated nature&V-FASTRʼs manual review requirement&依赖 -help&volunteer&AGGREGATION -large database&galaxy image&AGGREGATION -2015 january hart et al&rotation&AGGREGATION -project&date&依赖 -discovery&new interesting object&AGGREGATION -million&image&AGGREGATION -confirmation&important scientific hypothesis&AGGREGATION -formulation&new one&AGGREGATION -success&expectation&依赖 -project&success&依赖 -challenge&image classification&AGGREGATION -paradigm&v-fastr setting&依赖 -nature&archive&AGGREGATION -its&limits& -Galaxy Zoo&volunteer reviewer&依赖 -its&reviewers& -Galaxy Zoo&leisurely peruse&依赖 -University&Washington&AGGREGATION -it&“ crowd-sourced ” attempt&依赖 -protein and chain&wide range&依赖 -protein and chain&wide range&依赖 -protein and chain&human disease&依赖 -they&specific shape&依赖 -protein and chain&key role&依赖 -protein and chain&key role&依赖 -wide range&human disease&AGGREGATION -protein and chain&key role&依赖 -protein and chain&human disease&依赖 -their&function& -they&themselves&依赖 -chain&amino acid&AGGREGATION -protein and chain&human disease&依赖 -specific shape&function&依赖 -protein and chain&wide range&依赖 -researcher&challenge&依赖 -researcher&puzzle-solving capability&依赖 -researcher&challenge&依赖 -researcher&scale and complexity&依赖 -researcher&scale and complexity&依赖 -researcher&human being&依赖 -researcher&assistance&依赖 -researcher&assistance&依赖 -researcher&human being&依赖 -researcher&puzzle-solving capability&依赖 -scale and complexity&challenge&AGGREGATION -puzzle-solving capability&human being&AGGREGATION -player&arriving&依赖 -arrangement&total energy&依赖 -player&goal&依赖 -unknown and diverse strategy&human participant&AGGREGATION -its&participants& -Foldit&environment&依赖 -project&400,000 player&依赖 -implementation&metadata-driven framework&AGGREGATION -online review&V-FASTR candidate detection event&AGGREGATION -presentation&system architecture&AGGREGATION -our&methodology& -our&design& -consideration&design&依赖 -We&methodology&依赖 -their&imprint& -Development Methodology Several factor&development process&依赖 -development process&process&GENERALIZATION -need&VFASTR project&AGGREGATION -our&implementation& -aspect&design process&AGGREGATION -group&substantial experience&依赖 -group&broad range&实现 -our&group& -broad range&scientific domain&AGGREGATION -design and implementation&data system&AGGREGATION -group&design and implementation&依赖 -group&data system&实现 -close working relationship&success&依赖 -close working relationship&case&依赖 -success&project&AGGREGATION -close working relationship&project&依赖 -close working relationship&case&依赖 -close working relationship&success&依赖 -member&project science team&AGGREGATION -close working relationship&case&依赖 -close working relationship&success&依赖 -close working relationship&project&依赖 -close working relationship&project&依赖 -intuition&us&依赖 -intuition&software engineer&依赖 -our&intuitions& -intuition&software engineer&依赖 -intuition&software engineer&依赖 -intuition&us&依赖 -intuition&us&依赖 -technical challenge&system&AGGREGATION -it&V-FASTR science team&依赖 -it&member&依赖 -early identification&issue&AGGREGATION -our&communication& -member&V-FASTR science team&AGGREGATION -current system architecture&ongoing feedback loop&依赖 -current system architecture&science and software team&依赖 -direct result&ongoing feedback loop&AGGREGATION -part®ular third-party use&AGGREGATION -regular third-party use&VLBA instrument&AGGREGATION -experiment&“ guest ” status&依赖 -experiment&NRAO computing infrastructure&依赖 -experiment&“ guest ” status&依赖 -many&architectural decision&AGGREGATION -physical constraint&architectural decision&依赖 -physical constraint&many&依赖 -two type&product&AGGREGATION -data product&product&GENERALIZATION -hundred&file&AGGREGATION -V-FASTR data product&file&依赖 -V-FASTR data product&data product&GENERALIZATION -~ 800 job&~ 10 GB&AGGREGATION -storage&few week&依赖 -product&~ 10 – 20&依赖 -average rate&~ 10 – 20&AGGREGATION -product&average rate&依赖 -product&day (&依赖 -product&science team analyst&依赖 -candidate&review&依赖 -overview&average data volume&AGGREGATION -candidate&higher-resolution processing (&依赖 -core&design&AGGREGATION -network bandwidth constraint&host&AGGREGATION -network bandwidth constraint&us&依赖 -network bandwidth constraint&data transfer configuration&依赖 -complete transfer&raw , unreviewed , and possibly spurious detection datum&AGGREGATION -network bandwidth constraint&data transfer configuration&依赖 -network bandwidth constraint&us&依赖 -salient characteristic&candidate event&AGGREGATION -metada sufficient&candidate review framework&依赖 -candidate event&event&GENERALIZATION -careful selection process&beneficial side effect&依赖 -careful selection process&size&依赖 -careful selection process&allowing&依赖 -size&transferred product&AGGREGATION -longer retention period&JPL&依赖 -longer retention period&JPL&依赖 -security constraint&design&依赖 -system&two separate security domain&依赖 -system&NRAO and JPL&依赖 -security requirement&host system&AGGREGATION -functionality&possibility&依赖 -possibility&corruption&AGGREGATION -functionality&corruption&依赖 -functionality&corruption&依赖 -functionality&possibility&依赖 -software framework&framework&GENERALIZATION -we&consisting&依赖 -capture , transfer , and storage&metadata annotation&AGGREGATION -two principal component&capture , transfer , and storage&依赖 -web portal&portal&GENERALIZATION -we&software framework&依赖 -two principal component&metadata annotation&依赖 -V-FASTR data product&Metadata Pipeline&依赖 -V-FASTR data product&JPL side&依赖 -V-FASTR data product&metadata extraction and data archiving pipeline&依赖 -pipeline&three major software component&依赖 -pipeline&rsync&依赖 -Data product&NRAO staging area&依赖 -Data product&jpl server use rsync&依赖 -content&directory tree&AGGREGATION -jpl server use rsync&two server&依赖 -jpl server use rsync&minimal human intervention&依赖 -jpl server use rsync&content&依赖 -It&simplicity&依赖 -wide range&configuration option&AGGREGATION -its&simplicity& -file&compressed format&依赖 -client&product directory tree&依赖 -client&datum&依赖 -client&metada&依赖 -small subset&datum&AGGREGATION -client&small subset&依赖 -reduction&data product size&AGGREGATION -factor&3.5 � 103&AGGREGATION -average size&~ 35 GB&AGGREGATION -data product&JPL server&依赖 -OODT CAS Crawler daemon&new product&依赖 -they&OODT CAS Crawler daemon&依赖 -product directory&indicating&依赖 -named product&file manager catalog and ( 2 )&依赖 -product directory&special marker file&依赖 -no file&missing& -software component&component&GENERALIZATION -File Manager&project&依赖 -Depiction&volume estimate&依赖 -Depiction&job )&依赖 -Depiction&job )&依赖 -Depiction&volume estimate&依赖 -Depiction&volume estimate&依赖 -Depiction&job )&依赖 -Depiction&job )&依赖 -Depiction&full v-fastr datum flow&AGGREGATION -Depiction&volume estimate&依赖 -Depiction&volume estimate&依赖 -Depiction&job )&依赖 -candidate review framework&A and B&依赖 -candidate review framework&metada and derive product repository&依赖 -candidate review framework&metada and derive product repository&依赖 -candidate review framework&intersection&依赖 -intersection&A and B&AGGREGATION -candidate review framework&A and B&依赖 -candidate review framework&intersection&依赖 -rsync process&NRAO server&依赖 -Astronomical Journal 149:23 ( 7pp )&2015 january hart et al&依赖 -Astronomical Journal 149:23 ( 7pp )&2015 january hart et al&依赖 -Astronomical Journal 149:23 ( 7pp )&in-place&依赖 -Astronomical Journal 149:23 ( 7pp )&in-place&依赖 -rsync process&them&依赖 -we&OODT framework&依赖 -extensibility&OODT framework&AGGREGATION -we&extensibility&依赖 -OODT framework&framework&GENERALIZATION -three level&V-FASTR data product&依赖 -Information&job , scan , and event&依赖 -hierarchy&V-FASTR data product&AGGREGATION -Information&three level&依赖 -pair&available image&AGGREGATION -event&them&依赖 -File Manager&metada&依赖 -File Manager&back-end catalog&依赖 -File Manager&different object type&依赖 -its&catalog& -field&named key&依赖 -querying&information&AGGREGATION -decision&desire&依赖 -metada&single request&依赖 -metada&File Manager&依赖 -decision&querying&实现 -decision&web portal client&实现 -ingestion&dynamically name metadata field&AGGREGATION -second major component&candidate review framework&AGGREGATION -primary purpose&portal&AGGREGATION -location-independent perusal and assessment&potential candidate&AGGREGATION -top&Apache HTTPD Web Server&AGGREGATION -portal&Apache OODT Balance web framework&实现 -portal&running&实现 -portal&PHP web application&实现 -OODT Balance&its ability&依赖 -its&ability& -OODT Balance&integrate&依赖 -flexible , modular approach&web portal&依赖 -flexible , modular approach&framework&AGGREGATION -flexible , modular approach&web portal&依赖 -flexible , modular approach&us&依赖 -flexible , modular approach&us&依赖 -web portal&view&依赖 -web portal&available metada&依赖 -variety&view&AGGREGATION -web portal&variety&依赖 -web portal&view&依赖 -view&available metada&AGGREGATION -web portal&available metada&依赖 -web portal&variety&依赖 -job or run&multiple scan&依赖 -job or run&highest level&依赖 -layout&portal view&AGGREGATION -nature&signal&AGGREGATION -level&individual event candidate Figure 3&AGGREGATION -two graphical representation&nature&依赖 -two graphical representation&analyst&依赖 -two graphical representation&event&AGGREGATION -two graphical representation&analyst&依赖 -two graphical representation&signal&依赖 -two graphical representation&nature&依赖 -two graphical representation&signal&依赖 -part&initial candidate identification process&AGGREGATION -necessary structural clue&received signal&依赖 -product&RFI&AGGREGATION -image representation&an entire job ( many event&AGGREGATION -portal user&traditional , hierarchical navigation&依赖 -portal user&job&依赖 -metadata pipeline component&VFASTR candidate review framework&AGGREGATION -5&end-to-end framework&依赖 -5&capture , archiving&依赖 -5&end-to-end framework&依赖 -5&capture , archiving&依赖 -analysis process&process&GENERALIZATION -system&analysis process&依赖 -overall efficiency&project&AGGREGATION -online collaborative validation&fast transient candidate&AGGREGATION -candidate review framework&online collaborative validation&依赖 -we&previous section&依赖 -candidate review framework&model&依赖 -candidate review framework&fast transient candidate&依赖 -candidate review framework&fast transient candidate&依赖 -candidate review framework&model&依赖 -candidate review framework&model&依赖 -candidate review framework&online collaborative validation&依赖 -candidate review framework&fast transient candidate&依赖 -candidate review framework&online collaborative validation&依赖 -analyst&observational datum&依赖 -model&online collaborative validation&AGGREGATION -analyst&efficiency&依赖 -team&dispersed analyst&AGGREGATION -we&experience&依赖 -we&early result&依赖 -we&early result&依赖 -we&experience&依赖 -early result&experience&AGGREGATION -we&experience&依赖 -its&utility& -we&early result&依赖 -operational deployment&framework&AGGREGATION -we&early result&依赖 -we&experience&依赖 -evolution&tool&AGGREGATION -Experience The initial deployment&early summer 2012&依赖 -Experience The initial deployment&collaborative review framework&AGGREGATION -analyst&capability&依赖 -analyst&system&依赖 -capability&system&AGGREGATION -consequence&iterative feedback loop&AGGREGATION -process&updated release&依赖 -process&occurring&依赖 -first few month&deployment&AGGREGATION -part&analyst user&AGGREGATION -suggestion&addition&依赖 -number&mouse click&AGGREGATION -suggestion&various metadata field&依赖 -suggestion&various metadata field&依赖 -addition&various metadata field&AGGREGATION -element&web portal view&AGGREGATION -visual organization&element&AGGREGATION -suggestion&addition&依赖 -time&writing&AGGREGATION -V-FASTR portal&system&依赖 -V-FASTR portal&time&依赖 -we&usefulness&依赖 -V-FASTR portal&several week&依赖 -we&early conclusion&依赖 -V-FASTR portal&portal&GENERALIZATION -V-FASTR portal&writing&依赖 -we&system&依赖 -usefulness&system&AGGREGATION -V-FASTR portal&usefulness&依赖 -project&broad goal&依赖 -project&radio-transient event&依赖 -project&collaborative task&依赖 -its&goal& -their&tasks& -scientist&task&依赖 -online availability&greater flexibility&依赖 -online availability&greater flexibility&依赖 -online availability&greater flexibility&依赖 -online availability®ard&依赖 -online availability®ard&依赖 -online availability®ard&依赖 -online availability&greater flexibility&依赖 -online availability&data and metada&AGGREGATION -online availability®ard&依赖 -totality&candidate data and metada&AGGREGATION -anecdotal account&analyst&AGGREGATION -number&feature request&AGGREGATION -evolution&framework&AGGREGATION -roadmap&sort&AGGREGATION -interaction&browser&依赖 -’ activity&transition&依赖 -’ activity&transition&依赖 -interaction&browser&依赖 -science team&three key feature&依赖 -timely review&remaining& -timely review&detection candidate&AGGREGATION -review job&moment&依赖 -review job&email&依赖 -review job&analyst&依赖 -prioritized list&analystʼs outstanding review task&AGGREGATION -presentation&prioritized list&AGGREGATION -one person&analysis&依赖 -it&analysis load&依赖 -one person&fulltime job&依赖 -equitable scheduling&future review job&AGGREGATION -analysis contribution&individual user&AGGREGATION -screen shot&initial version&AGGREGATION -initial version&web portal component&AGGREGATION -image metada&full metadata listing&依赖 -image metada&individual event candidate&依赖 -portal home page&recent job&依赖 -last step&associated raw datum&依赖 -analyst&a candidate event merit&依赖 -analyst&high-resolution followup&依赖 -it&later date&依赖 -analyst&NRAO machine&依赖 -certain defined activity&portal environment&依赖 -CONCLUSION V-FASTR&V-FASTR&GENERALIZATION -CONCLUSION V-FASTR&particularly challenging experiment&依赖 -data flow&ranging&依赖 -data flow&transport mechanism&依赖 -manual review&place&依赖 -manual review&analyst&依赖 -data flow&multiple physical location&依赖 -manual review&three continent )&依赖 -Various component&millisecond and hourly&依赖 -Various component&millisecond and hourly&依赖 -Various component&system&AGGREGATION -human expert&scheduled pattern recognition engine&依赖 -human expert&own role&依赖 -data processing component&addition&依赖 -human expert&overall architecture&依赖 -their&role& -datum&system&依赖 -V-FASTR portal&critical role&依赖 -effort&Jet Propulsion Laboratory&依赖 -Interoperable Data Architecture&et al. 2011&依赖 -Interoperable Data Architecture&et al. 2011&依赖 -Interoperable Data Architecture&Computer-Based Medical Systems&依赖 -Interoperable Data Architecture&Computer-Based Medical Systems&依赖 -Interoperable Data Architecture&Computer-Based Medical Systems&依赖 -Interoperable Data Architecture&Computer-Based Medical Systems&依赖 -Interoperable Data Architecture&et al. 2011&依赖 -Interoperable Data Architecture&Computer-Based Medical Systems&依赖 -Interoperable Data Architecture&Computer-Based Medical Systems&依赖 -Interoperable Data Architecture&Computer-Based Medical Systems&依赖 -Interoperable Data Architecture&Computer-Based Medical Systems&依赖 -Interoperable Data Architecture&et al. 2011&依赖 -Interoperable Data Architecture&et al. 2011&依赖 -Interoperable Data Architecture&Computer-Based Medical Systems&依赖 -Interoperable Data Architecture&et al. 2011&依赖 -Interoperable Data Architecture&Computer-Based Medical Systems&依赖 -Interoperable Data Architecture&et al. 2011&依赖 -Interoperable Data Architecture&Computer-Based Medical Systems&依赖 -Interoperable Data Architecture&et al. 2011&依赖 -Interoperable Data Architecture&Computer-Based Medical Systems&依赖 -Interoperable Data Architecture&et al. 2011&依赖 -Interoperable Data Architecture&Computer-Based Medical Systems&依赖 -Interoperable Data Architecture&Computer-Based Medical Systems&依赖 -Interoperable Data Architecture&Computer-Based Medical Systems&依赖 -Interoperable Data Architecture&et al. 2011&依赖 -Interoperable Data Architecture&et al. 2011&依赖 -Interoperable Data Architecture&et al. 2011&依赖 -Interoperable Data Architecture&et al. 2011&依赖 -Interoperable Data Architecture&et al. 2011&依赖 -Interoperable Data Architecture&et al. 2011&依赖 -A Software Architecture-based Framework&Software Engineering&依赖 -A Software Architecture-based Framework&Software Engineering&依赖 -A Software Architecture-based Framework&Software Engineering&依赖 -A Software Architecture-based Framework&acm ) , 721 – 30 mattmann , c. , freeborn , d. , crichton , d. , et al. 2009&依赖 -A Software Architecture-based Framework&acm ) , 721 – 30 mattmann , c. , freeborn , d. , crichton , d. , et al. 2009&依赖 -A Software Architecture-based Framework&Software Engineering&依赖 -A Software Architecture-based Framework&Software Engineering&依赖 -A Software Architecture-based Framework&acm ) , 721 – 30 mattmann , c. , freeborn , d. , crichton , d. , et al. 2009&依赖 -A Software Architecture-based Framework&Software Engineering&依赖 -A Software Architecture-based Framework&acm ) , 721 – 30 mattmann , c. , freeborn , d. , crichton , d. , et al. 2009&依赖 -A Software Architecture-based Framework&Software Engineering&依赖 -A Software Architecture-based Framework&Software Engineering&依赖 -A Software Architecture-based Framework&acm ) , 721 – 30 mattmann , c. , freeborn , d. , crichton , d. , et al. 2009&依赖 -A Software Architecture-based Framework&Software Engineering&依赖 -A Software Architecture-based Framework&Software Engineering&依赖 -A Software Architecture-based Framework&acm ) , 721 – 30 mattmann , c. , freeborn , d. , crichton , d. , et al. 2009&依赖 -A Software Architecture-based Framework&Software Engineering&依赖 -A Software Architecture-based Framework&acm ) , 721 – 30 mattmann , c. , freeborn , d. , crichton , d. , et al. 2009&依赖 -A Software Architecture-based Framework&acm ) , 721 – 30 mattmann , c. , freeborn , d. , crichton , d. , et al. 2009&依赖 -A Software Architecture-based Framework&Software Engineering&依赖 -A Software Architecture-based Framework&acm ) , 721 – 30 mattmann , c. , freeborn , d. , crichton , d. , et al. 2009&依赖 -A Software Architecture-based Framework&acm ) , 721 – 30 mattmann , c. , freeborn , d. , crichton , d. , et al. 2009&依赖 -A Software Architecture-based Framework&acm ) , 721 – 30 mattmann , c. , freeborn , d. , crichton , d. , et al. 2009&依赖 -A Software Architecture-based Framework&Software Engineering&依赖 -A Software Architecture-based Framework&acm ) , 721 – 30 mattmann , c. , freeborn , d. , crichton , d. , et al. 2009&依赖 -A Software Architecture-based Framework&acm ) , 721 – 30 mattmann , c. , freeborn , d. , crichton , d. , et al. 2009&依赖 -A Software Architecture-based Framework&Software Engineering&依赖 -A Software Architecture-based Framework&acm ) , 721 – 30 mattmann , c. , freeborn , d. , crichton , d. , et al. 2009&依赖 -A Software Architecture-based Framework&acm ) , 721 – 30 mattmann , c. , freeborn , d. , crichton , d. , et al. 2009&依赖 -A Software Architecture-based Framework&Software Engineering&依赖 -npp sounder peate missions ( piscataway&Information Technology&依赖 -npp sounder peate missions ( piscataway&space mission challenge&依赖 -npp sounder peate missions ( piscataway&Information Technology&依赖 -npp sounder peate missions ( piscataway&Information Technology&依赖 -npp sounder peate missions ( piscataway&Information Technology&依赖 -npp sounder peate missions ( piscataway&Information Technology&依赖 -npp sounder peate missions ( piscataway&Information Technology&依赖 -npp sounder peate missions ( piscataway&space mission challenge&依赖 -npp sounder peate missions ( piscataway&Information Technology&依赖 -npp sounder peate missions ( piscataway&space mission challenge&依赖 -npp sounder peate missions ( piscataway&space mission challenge&依赖 -npp sounder peate missions ( piscataway&Information Technology&依赖 -acm ) , 36 – 42 wayth , r. ,&2015 january hart et al&依赖 -2015 january hart et al&II International Workshop&AGGREGATION -acm ) , 36 – 42 wayth , r. ,&Software Engineering&依赖 -acm ) , 36 – 42 wayth , r. ,&2015 january hart et al&依赖 -acm ) , 36 – 42 wayth , r. ,&cloud computing ( new york&依赖 -acm ) , 36 – 42 wayth , r. ,&cloud computing ( new york&依赖 -acm ) , 36 – 42 wayth , r. ,&Software Engineering&依赖 -acm ) , 36 – 42 wayth , r. ,&II International Workshop&依赖 -acm ) , 36 – 42 wayth , r. ,&II International Workshop&依赖 -acm ) , 36 – 42 wayth , r. ,&II International Workshop&依赖 -acm ) , 36 – 42 wayth , r. ,&II International Workshop&依赖 -acm ) , 36 – 42 wayth , r. ,&cloud computing ( new york&依赖 -acm ) , 36 – 42 wayth , r. ,&2015 january hart et al&依赖 -acm ) , 36 – 42 wayth , r. ,&2015 january hart et al&依赖 -acm ) , 36 – 42 wayth , r. ,&Software Engineering&依赖 -acm ) , 36 – 42 wayth , r. ,&Software Engineering&依赖 -acm ) , 36 – 42 wayth , r. ,&cloud computing ( new york&依赖 diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/A FRAMEWORK FOR COLLABORATIVE REVIEW OF CANDIDATE EVENTS IN HIGH DATA RATE STREAMS THE V-FASTR EXPERIMENT AS A CASE STUDY-simEnts.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/A FRAMEWORK FOR COLLABORATIVE REVIEW OF CANDIDATE EVENTS IN HIGH DATA RATE STREAMS THE V-FASTR EXPERIMENT AS A CASE STUDY-simEnts.txt deleted file mode 100644 index 0e2538191b3b77492c1474355130619c96ca760f..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/doc/Apache OODT File Manager/A FRAMEWORK FOR COLLABORATIVE REVIEW OF CANDIDATE EVENTS IN HIGH DATA RATE STREAMS THE V-FASTR EXPERIMENT AS A CASE STUDY-simEnts.txt +++ /dev/null @@ -1,694 +0,0 @@ -structs,structs -BooleanQueryCriteria,BooleanQueryCriteria -Element,Element -ExtractorSpec,ExtractorSpec -ExtractorSpec,spicified Extractor extractors -ExtractorSpec,MimeTypeExtractor -ExtractorSpec,CoreMetExtractor -ExtractorSpec,JobSpec -FileTransferStatus,FileTransferStatus -FileTransferStatus,File Manager extension points -FileTransferStatus,FileManagerDashboardNulled -FileTransferStatus,DevExtremeJavaScriptFile -FileTransferStatus,File Manager UI component -FileTransferStatus,File Manager server interface -FileTransferStatus,File Manager Command class -FileTransferStatus,File Manager web interface -FileTransferStatus,FileManagerAppUI -FileTransferStatus,File content -FileTransferStatus,JSXFile -FileTransferStatus,File Manager file System Provider -FileTransferStatus,File type -FileTransferStatus,File information -FreeTextQueryCriteria,FreeTextQueryCriteria -FreeTextQueryCriteria,FreePastry -FreeTextQueryCriteria,implementation Free -Product,Product -ProductPage,ProductPage -ProductPage,Product structures -ProductPage,ProductMetadata -ProductPage,ProductGenerationExecutives -ProductPage,ProductPctTransferred -ProductPage,ProductId -ProductPage,FileManagerPage -ProductPage,ProductTypes -ProductPage,ProductStructure -ProductPage,Product operation -ProductPage,Product instance information -ProductPage,Product listing pages -ProductType,ProductType -ProductType,Product structures -ProductType,ProductMetadata -ProductType,Type Script script -ProductType,ProductGenerationExecutives -ProductType,ProductPctTransferred -ProductType,ProductId -ProductType,ProductTypes -ProductType,ProductStructure -ProductType,Product operation -ProductType,Product instance information -ProductType,product Type -ProductType,Product listing pages -Query,Query -QueryCriteria,QueryCriteria -RangeQueryCriteria,RangeQueryCriteria -Reference,Reference -TermQueryCriteria,TermQueryCriteria -TestProduct,TestProduct -TestReference,TestReference -System,System -XmlRpcFileManager,XmlRpcFileManager -XmlRpcFileManager,import File Manager -XmlRpcFileManager,OpenFileManager -XmlRpcFileManager,XmlRpcFileManagerClient -XmlRpcFileManager,TrilloFileManager -XmlRpcFileManager,KendoUIFileManager -XmlRpcFileManager,Manager The Repository Manager extension -XmlRpcFileManager,ExtremeJavaScriptFileManager -XmlRpcFileManager,RepositoryManagerTheRepositoryManager -XmlRpcFileManager,CASFileManager -XmlRpcFileManager,Xml Rpc File Manager complex -XmlRpcFileManager,Manager extension point -XmlRpcFileManager,WebixFileManager -XmlRpcFileManager,bs4 File Manager -XmlRpcFileManager,ApacheOODTFileManager -XmlRpcFileManagerClient,XmlRpcFileManagerClient -XmlRpcFileManagerClient,Client implementation -XmlRpcFileManagerClient,Client React connector -XmlRpcFileManagerClient,Client React component -XmlRpcFileManagerClient,Client side metadata -XmlRpcFileManagerClient,MahasenClient -Dispatcher,Dispatcher -Result,Result -SecureWebServer,SecureWebServer -SecureWebServer,ApacheHTTPDWebServer -SecureWebServer,Server Node API v1 Localization -SecureWebServer,WindowsServer -Repository,Repository -RepositoryManager,RepositoryManager -RepositoryManager,SoftwarePackageManager -RepositoryManager,TaskManager -RepositoryManager,import File Manager -RepositoryManager,OpenFileManager -RepositoryManager,NETCoreFileManager -RepositoryManager,OODTCASFileManager -RepositoryManager,NodePackageManager -RepositoryManager,TrilloFileManager -RepositoryManager,KendoUIFileManager -RepositoryManager,WordPressFileManager -RepositoryManager,Manager The Repository Manager extension -RepositoryManager,ManagerIngestUseCase -RepositoryManager,ExtremeJavaScriptFileManager -RepositoryManager,Repository Manager extension point -RepositoryManager,RepositoryManagerTheRepositoryManager -RepositoryManager,RepositoryManagerTheRepositoryManager -RepositoryManager,CASFileManager -RepositoryManager,ManagerObjectModel -RepositoryManager,demand Library Manager -RepositoryManager,DeployTrilloFileManager -RepositoryManager,Manager extension point -RepositoryManager,WebixFileManager -RepositoryManager,bs4 File Manager -RepositoryManager,ApacheOODTFileManager -RepositoryManagerFactory,RepositoryManagerFactory -RepositoryManagerFactory,Repository Manager extension point -RepositoryManagerFactory,RepositoryManagerTheRepositoryManager -DataSourceRepositoryManager,DataSourceRepositoryManager -DataSourceRepositoryManager,SoftwarePackageManager -DataSourceRepositoryManager,Data nodes -DataSourceRepositoryManager,Manager control -DataSourceRepositoryManager,TaskManager -DataSourceRepositoryManager,import File Manager -DataSourceRepositoryManager,OpenFileManager -DataSourceRepositoryManager,NETCoreFileManager -DataSourceRepositoryManager,OODTCASFileManager -DataSourceRepositoryManager,NodePackageManager -DataSourceRepositoryManager,MoneydanceFinancialData -DataSourceRepositoryManager,Data Transfer interface -DataSourceRepositoryManager,Manager The Repository Manager extension -DataSourceRepositoryManager,Data products -DataSourceRepositoryManager,ExtremeJavaScriptFileManager -DataSourceRepositoryManager,ManagerArchitecture -DataSourceRepositoryManager,Data registry -DataSourceRepositoryManager,Manager catalog -DataSourceRepositoryManager,RepositoryManagerTheRepositoryManager -DataSourceRepositoryManager,CASFileManager -DataSourceRepositoryManager,ManagerObjectModel -DataSourceRepositoryManager,demand Library Manager -DataSourceRepositoryManager,Manager extension point -DataSourceRepositoryManager,WebixFileManager -DataSourceRepositoryManager,DataManagementSystems -DataSourceRepositoryManager,Data volume -DataSourceRepositoryManager,DataSystemDevelopment -DataSourceRepositoryManagerFactory,DataSourceRepositoryManagerFactory -DataSourceRepositoryManagerFactory,Data nodes -DataSourceRepositoryManagerFactory,MoneydanceFinancialData -DataSourceRepositoryManagerFactory,Data Transfer extension point -DataSourceRepositoryManagerFactory,Data Transfer interface -DataSourceRepositoryManagerFactory,Data products -DataSourceRepositoryManagerFactory,Data registry -DataSourceRepositoryManagerFactory,DataGridManagementSystem -DataSourceRepositoryManagerFactory,DataManagementSystems -DataSourceRepositoryManagerFactory,Data volume -DataSourceRepositoryManagerFactory,DataSystemDevelopment -ScienceDataRepositoryManager,ScienceDataRepositoryManager -ScienceDataRepositoryManager,SoftwarePackageManager -ScienceDataRepositoryManager,Manager control -ScienceDataRepositoryManager,TaskManager -ScienceDataRepositoryManager,import File Manager -ScienceDataRepositoryManager,OpenFileManager -ScienceDataRepositoryManager,NETCoreFileManager -ScienceDataRepositoryManager,OODTCASFileManager -ScienceDataRepositoryManager,SciencePgeConfigFileWriter -ScienceDataRepositoryManager,NodePackageManager -ScienceDataRepositoryManager,TrilloFileManager -ScienceDataRepositoryManager,KendoUIFileManager -ScienceDataRepositoryManager,WordPressFileManager -ScienceDataRepositoryManager,Manager The Repository Manager extension -ScienceDataRepositoryManager,ComputerScience -ScienceDataRepositoryManager,ExtremeJavaScriptFileManager -ScienceDataRepositoryManager,ManagerArchitecture -ScienceDataRepositoryManager,ScienceProduct -ScienceDataRepositoryManager,Manager catalog -ScienceDataRepositoryManager,RepositoryManagerTheRepositoryManager -ScienceDataRepositoryManager,CASFileManager -ScienceDataRepositoryManager,ManagerObjectModel -ScienceDataRepositoryManager,SpaceScience -ScienceDataRepositoryManager,demand Library Manager -ScienceDataRepositoryManager,DeployTrilloFileManager -ScienceDataRepositoryManager,Manager extension point -ScienceDataRepositoryManager,WebixFileManager -ScienceDataRepositoryManager,bs4 File Manager -ScienceDataRepositoryManager,ApacheOODTFileManager -ScienceDataRepositoryManagerFactory,ScienceDataRepositoryManagerFactory -ScienceDataRepositoryManagerFactory,ComputerScience -ScienceDataRepositoryManagerFactory,ScienceProduct -ScienceDataRepositoryManagerFactory,SpaceScience -ScienceDataRepositoryManagerFactory,ScienceDataProcessingSystems -TestXMLRepositoryManager,TestXMLRepositoryManager -TestXMLRepositoryManager,SoftwarePackageManager -TestXMLRepositoryManager,Manager control -TestXMLRepositoryManager,TaskManager -TestXMLRepositoryManager,import File Manager -TestXMLRepositoryManager,OpenFileManager -TestXMLRepositoryManager,NodePackageManager -TestXMLRepositoryManager,TrilloFileManager -TestXMLRepositoryManager,Manager The Repository Manager extension -TestXMLRepositoryManager,ExtremeJavaScriptFileManager -TestXMLRepositoryManager,ManagerArchitecture -TestXMLRepositoryManager,Manager catalog -TestXMLRepositoryManager,RepositoryManagerTheRepositoryManager -TestXMLRepositoryManager,CASFileManager -TestXMLRepositoryManager,ManagerObjectModel -TestXMLRepositoryManager,demand Library Manager -TestXMLRepositoryManager,Manager extension point -TestXMLRepositoryManager,WebixFileManager -TestXMLRepositoryManager,bs4 File Manager -XMLRepositoryManager,XMLRepositoryManager -XMLRepositoryManager,XMLMetadataConceptCatalog -XMLRepositoryManager,Manager control -XMLRepositoryManager,TaskManager -XMLRepositoryManager,import File Manager -XMLRepositoryManager,OpenFileManager -XMLRepositoryManager,NETCoreFileManager -XMLRepositoryManager,OODTCASFileManager -XMLRepositoryManager,XML files -XMLRepositoryManager,TrilloFileManager -XMLRepositoryManager,KendoUIFileManager -XMLRepositoryManager,WordPressFileManager -XMLRepositoryManager,Manager The Repository Manager extension -XMLRepositoryManager,ManagerIngestUseCase -XMLRepositoryManager,ExtremeJavaScriptFileManager -XMLRepositoryManager,ManagerArchitecture -XMLRepositoryManager,Manager catalog -XMLRepositoryManager,RepositoryManagerTheRepositoryManager -XMLRepositoryManager,CASFileManager -XMLRepositoryManager,DeployTrilloFileManager -XMLRepositoryManager,WebixFileManager -XMLRepositoryManager,bs4 File Manager -XMLRepositoryManager,ApacheOODTFileManager -XMLRepositoryManagerFactory,XMLRepositoryManagerFactory -XMLRepositoryManagerFactory,XML files -Metadata,Metadata -CoreMetKeys,CoreMetKeys -CoreMetKeys,Core machines -CoreMetKeys,CoreData -FileAttributesMetKeys,FileAttributesMetKeys -FileAttributesMetKeys,File Managers functionality -FileAttributesMetKeys,File Browser dialogs -FileAttributesMetKeys,FileRetrievalSystem -FileAttributesMetKeys,FileUploadComponent -FileAttributesMetKeys,DevExtremeJavaScriptFile -FileAttributesMetKeys,FileLocationExtractor -FileAttributesMetKeys,FilePctTransferred -FileAttributesMetKeys,File Repository layouts -FileAttributesMetKeys,File content -FileAttributesMetKeys,JSXFile -FileAttributesMetKeys,File type -ProductMetKeys,ProductMetKeys -ProductMetKeys,Product structures -ProductMetKeys,ProductMetadata -ProductMetKeys,ProductId -ProductMetKeys,ProductTypes -ProductMetKeys,ScienceProduct -ProductMetKeys,ProductStructure -ProductMetKeys,Product operation -FilemgrMetExtractor,FilemgrMetExtractor -FilemgrMetExtractor,CoreMetExtractor -AbstractFilemgrMetExtractor,AbstractFilemgrMetExtractor -AbstractFilemgrMetExtractor,MimeTypeExtractor -AbstractFilemgrMetExtractor,CoreMetExtractor -CoreMetExtractor,CoreMetExtractor -CoreMetExtractor,Core machines -CoreMetExtractor,Extractor extractors -CoreMetExtractor,MimeTypeExtractor -CoreMetExtractor,CoreData -TestAbstractFilemgrMetExtractor,TestAbstractFilemgrMetExtractor -TestAbstractFilemgrMetExtractor,Extractor extractors -TestAbstractFilemgrMetExtractor,MimeTypeExtractor -TestAbstractFilemgrMetExtractor,CoreMetExtractor -TestCoreMetExtractor,TestCoreMetExtractor -TestCoreMetExtractor,Extractor extractors -TestCoreMetExtractor,MimeTypeExtractor -TestCoreMetExtractor,CoreMetExtractor -Catalog,Catalog -CatalogFactory,CatalogFactory -DataSourceCatalog,DataSourceCatalog -DataSourceCatalog,XMLMetadataConceptCatalog -DataSourceCatalog,Data nodes -DataSourceCatalog,LuceneIndexCatalog -DataSourceCatalog,Catalog extension point interface -DataSourceCatalog,Data Transfer extension point -DataSourceCatalog,Data products -DataSourceCatalog,DataNodes -DataSourceCatalog,Data registry -DataSourceCatalog,CoreData -DataSourceCatalog,DataSystemPDS4InformationModel-Driven -DataSourceCatalog,LuceneCatalog -DataSourceCatalog,DataList -DataSourceCatalog,Catalog interface -DataSourceCatalog,GoddardEarthScienceData -DataSourceCatalog,DataStructure -DataSourceCatalog,DataGridManagementSystem -DataSourceCatalog,Data volume -DataSourceCatalogFactory,DataSourceCatalogFactory -DataSourceCatalogFactory,Data nodes -DataSourceCatalogFactory,DataIntensiveScientific -DataSourceCatalogFactory,MoneydanceFinancialData -DataSourceCatalogFactory,Data Transfer interface -DataSourceCatalogFactory,Data products -DataSourceCatalogFactory,DataNodes -DataSourceCatalogFactory,Data registry -DataSourceCatalogFactory,CoreData -DataSourceCatalogFactory,DataSystemPDS4InformationModel-Driven -DataSourceCatalogFactory,DataList -DataSourceCatalogFactory,DataStructure -DataSourceCatalogFactory,DataManagementSystems -DataSourceCatalogFactory,Data volume -DataSourceCatalogFactory,DataSystemDevelopment -LenientDataSourceCatalog,LenientDataSourceCatalog -LenientDataSourceCatalog,LuceneIndexCatalog -LenientDataSourceCatalog,AMGAMetadataCatalog -LenientDataSourceCatalog,Catalog extension point -LenientDataSourceCatalog,LuceneCatalog -LenientDataSourceCatalog,Catalog interface -LenientDataSourceCatalog,ApacheOODTCatalog -LuceneCatalog,LuceneCatalog -LuceneCatalog,XMLMetadataConceptCatalog -LuceneCatalog,LuceneIndexCatalog -LuceneCatalog,LuceneIndexCatalog -LuceneCatalog,AMGAMetadataCatalog -LuceneCatalog,Catalog extension point interface -LuceneCatalog,Catalog extension point -LuceneCatalog,ApacheOODTCatalog -LuceneCatalog,default Apache Lucene -LuceneCatalogFactory,LuceneCatalogFactory -LuceneCatalogFactory,LuceneCatalog -MappedDataSourceCatalog,MappedDataSourceCatalog -MappedDataSourceCatalog,LuceneIndexCatalog -MappedDataSourceCatalog,AMGAMetadataCatalog -MappedDataSourceCatalog,Catalog extension point -MappedDataSourceCatalog,LuceneCatalog -MappedDataSourceCatalog,Catalog interface -MappedDataSourceCatalog,ApacheOODTCatalog -MappedDataSourceCatalogFactory,MappedDataSourceCatalogFactory -MockCatalog,MockCatalog -MockCatalogFactory,MockCatalogFactory -ScienceDataCatalog,ScienceDataCatalog -ScienceDataCatalog,XMLMetadataConceptCatalog -ScienceDataCatalog,AMGAMetadataCatalog -ScienceDataCatalog,SciencePgeConfigFileWriter -ScienceDataCatalog,Catalog extension point interface -ScienceDataCatalog,ComputerScience -ScienceDataCatalog,ScienceProduct -ScienceDataCatalog,LuceneCatalog -ScienceDataCatalog,Catalog interface -ScienceDataCatalog,ApacheOODTCatalog -ScienceDataCatalog,SpaceScience -ScienceDataCatalog,ScienceDataProcessingSystems -ScienceDataCatalogFactory,ScienceDataCatalogFactory -ScienceDataCatalogFactory,SciencePgeConfigFileWriter -ScienceDataCatalogFactory,ComputerScience -ScienceDataCatalogFactory,ScienceProduct -ScienceDataCatalogFactory,SpaceScience -Catalog,Catalog -ProductIdGenerator,ProductIdGenerator -ProductIdGenerator,Product structures -ProductIdGenerator,ProductId -ProductIdGenerator,ProductTypes -ProductIdGenerator,ScienceProduct -ProductIdGenerator,ProductStructure -ProductIdGenerator,Product operation -ProductSerializer,ProductSerializer -CompleteProduct,CompleteProduct -CompleteProduct,ProductGenerationExecutives -CompleteProduct,ProductPctTransferred -CompleteProduct,ScienceProduct -CompleteProduct,Product instance information -CompleteProduct,Product listing pages -DefaultProductSerializer,DefaultProductSerializer -DefaultProductSerializer,Default configuration -NameProductIdGenerator,NameProductIdGenerator -NameProductIdGenerator,NameNode -Parameters,Parameters -QueryResponse,QueryResponse -SolrCatalog,SolrCatalog -SolrCatalogFactory,SolrCatalogFactory -SolrClient,SolrClient -UUIDProductIdGenerator,UUIDProductIdGenerator -Validation,Validation -ValidationLayer,ValidationLayer -ValidationLayer,ValidationLayerTheValidationLayer -ValidationLayer,ValidationLayerTheValidationLayer -ValidationLayer,Validation Layer extension point -ValidationLayer,Layer extension point -ValidationLayer,Layer The Validation Layer extension -ValidationLayerFactory,ValidationLayerFactory -ValidationLayerFactory,ValidationLayerTheValidationLayer -ValidationLayerFactory,Validation Layer extension point -DataSourceValidationLayer,DataSourceValidationLayer -DataSourceValidationLayer,Data nodes -DataSourceValidationLayer,DataIntensiveScientific -DataSourceValidationLayer,DataView -DataSourceValidationLayer,MoneydanceFinancialData -DataSourceValidationLayer,ValidationLayerTheValidationLayer -DataSourceValidationLayer,Data products -DataSourceValidationLayer,DataNodes -DataSourceValidationLayer,Data registry -DataSourceValidationLayer,CoreData -DataSourceValidationLayer,DataSystemPDS4InformationModel-Driven -DataSourceValidationLayer,DataList -DataSourceValidationLayer,DataSource -DataSourceValidationLayer,DataProtocols -DataSourceValidationLayer,Layer extension point -DataSourceValidationLayer,DataStructure -DataSourceValidationLayer,Layer The Validation Layer extension -DataSourceValidationLayer,DataGridManagementSystem -DataSourceValidationLayer,DataManagementSystems -DataSourceValidationLayer,Data volume -DataSourceValidationLayer,DataSystemDevelopment -DataSourceValidationLayerFactory,DataSourceValidationLayerFactory -DataSourceValidationLayerFactory,Data nodes -DataSourceValidationLayerFactory,DataIntensiveScientific -DataSourceValidationLayerFactory,DataView -DataSourceValidationLayerFactory,MoneydanceFinancialData -DataSourceValidationLayerFactory,Data Transfer extension point -DataSourceValidationLayerFactory,Data products -DataSourceValidationLayerFactory,DataNodes -DataSourceValidationLayerFactory,Data registry -DataSourceValidationLayerFactory,CoreData -DataSourceValidationLayerFactory,DataList -DataSourceValidationLayerFactory,GoddardEarthScienceData -DataSourceValidationLayerFactory,DataSource -DataSourceValidationLayerFactory,DataProtocols -DataSourceValidationLayerFactory,DataStructure -DataSourceValidationLayerFactory,DataGridManagementSystem -DataSourceValidationLayerFactory,DataManagementSystems -DataSourceValidationLayerFactory,Data volume -DataSourceValidationLayerFactory,DataSystemDevelopment -ScienceDataValidationLayer,ScienceDataValidationLayer -ScienceDataValidationLayer,SciencePgeConfigFileWriter -ScienceDataValidationLayer,ValidationLayerTheValidationLayer -ScienceDataValidationLayer,ComputerScience -ScienceDataValidationLayer,ScienceProduct -ScienceDataValidationLayer,GameScience -ScienceDataValidationLayer,SpaceScience -ScienceDataValidationLayer,Layer extension point -ScienceDataValidationLayer,ScienceDataProcessingSystems -ScienceDataValidationLayer,Layer The Validation Layer extension -ScienceDataValidationLayerFactory,ScienceDataValidationLayerFactory -ScienceDataValidationLayerFactory,ComputerScience -ScienceDataValidationLayerFactory,ScienceProduct -ScienceDataValidationLayerFactory,GameScience -ScienceDataValidationLayerFactory,SpaceScience -ScienceDataValidationLayerFactory,ScienceDataProcessingSystems -TestXMLValidationLayer,TestXMLValidationLayer -TestXMLValidationLayer,ValidationLayerTheValidationLayer -TestXMLValidationLayer,Layer extension point -TestXMLValidationLayer,Layer The Validation Layer extension -XMLValidationLayer,XMLValidationLayer -XMLValidationLayer,XMLMetadataConceptCatalog -XMLValidationLayer,XML files -XMLValidationLayer,ValidationLayerTheValidationLayer -XMLValidationLayer,XML Metadata Concept catalog -XMLValidationLayer,XML syntax -XMLValidationLayer,Layer The Validation Layer extension -XMLValidationLayerFactory,XMLValidationLayerFactory -XMLValidationLayerFactory,XML files -XMLValidationLayerFactory,XML syntax -versioning,versioning -Versioner,Versioner -Versioner,class AcquisitionDateVersioner -Versioner,class BasicVersioner -Versioner,class ConfigurableMetadataBasedFileVersioner -Versioner,class DateTimeVersioner -Versioner,class DirectoryProductVersioner -Versioner,class InPlaceVersioner -Versioner,class MetadataBasedFileVersioner -Versioner,class ProductTypeMetVersioner -Versioner,class SingleFileBasicVersioner -AcquisitionDateVersioner,AcquisitionDateVersioner -BasicVersioner,BasicVersioner -BasicVersioner,Versioner extension point -ConfigurableMetadataBasedFileVersioner,ConfigurableMetadataBasedFileVersioner -ConfigurableMetadataBasedFileVersioner,Versioner extension point -DateTimeVersioner,DateTimeVersioner -DirectoryProductVersioner,DirectoryProductVersioner -InPlaceVersioner,InPlaceVersioner -MetadataBasedFileVersioner,MetadataBasedFileVersioner -MetadataBasedFileVersioner,Versioner extension point -MetadataBasedFileVersioner,ProductMetadata -MetadataBasedFileVersioner,Metadata objects -MetadataBasedFileVersioner,Metadata information -MetadataBasedFileVersioner,Metadata management -ProductTypeMetVersioner,ProductTypeMetVersioner -ProductTypeMetVersioner,Versioner extension point -ProductTypeMetVersioner,Product structures -ProductTypeMetVersioner,ProductMetadata -ProductTypeMetVersioner,ProductGenerationExecutives -ProductTypeMetVersioner,ProductPctTransferred -ProductTypeMetVersioner,ProductId -ProductTypeMetVersioner,ProductTypes -ProductTypeMetVersioner,ScienceProduct -ProductTypeMetVersioner,ProductStructure -ProductTypeMetVersioner,Product operation -ProductTypeMetVersioner,Product instance information -ProductTypeMetVersioner,Product listing pages -SingleFileBasicVersioner,SingleFileBasicVersioner -SingleFileBasicVersioner,Versioner extension point -TestAcquisitionDateVersioner,TestAcquisitionDateVersioner -TestBasicVersioner,TestBasicVersioner -TestConfigurableMetadataBasedFileVersioner,TestConfigurableMetadataBasedFileVersioner -TestConfigurableMetadataBasedFileVersioner,Versioner extension point -TestDateTimeVersioner,TestDateTimeVersioner -TestDirectoryBasedProductVersioner,TestDirectoryBasedProductVersioner -TestDirectoryBasedProductVersioner,Versioner extension point -TestInPlaceVersioner,TestInPlaceVersioner -TestInPlaceVersioner,Versioner extension point -TestMetadataBasedFileVersioner,TestMetadataBasedFileVersioner -TestMetadataBasedFileVersioner,Versioner extension point -TestProductTypeMetVersioner,TestProductTypeMetVersioner -TestProductTypeMetVersioner,Versioner extension point -TestSingleFileBasicVersioner,TestSingleFileBasicVersioner -TestSingleFileBasicVersioner,Versioner extension point -VersioningUtils,VersioningUtils -ingest,ingest -Cache,Cache -CacheFactory,CacheFactory -Ingester,Ingester -RemoteableCache,RemoteableCache -AbstractCacheServerFactory,AbstractCacheServerFactory -CachedIngester,CachedIngester -CmdLineIngester,CmdLineIngester -LocalCache,LocalCache -LocalCacheFactory,LocalCacheFactory -RmiCache,RmiCache -RmiCacheFactory,RmiCacheFactory -RmiCacheServer,RmiCacheServer -RmiCacheServerFactory,RmiCacheServerFactory -StdIngester,StdIngester -TestCachedIngester,TestCachedIngester -TestLocalCache,TestLocalCache -TestRmiCache,TestRmiCache -TestStdIngester,TestStdIngester -datatransfer,datatransfer -DataTransfer,DataTransfer -DataTransfer,Data nodes -DataTransfer,DataIntensiveScientific -DataTransfer,LocalDataTransfer -DataTransfer,client Transfer -DataTransfer,MoneydanceFinancialData -DataTransfer,PlaceDataTransfer -DataTransfer,Data Transfer extension point -DataTransfer,Data Transfer interface -DataTransfer,Data products -DataTransfer,DataNodes -DataTransfer,RemoteDataTransfer -DataTransfer,Data registry -DataTransfer,DataSystemPDS4InformationModel-Driven -DataTransfer,DataList -DataTransfer,GoddardEarthScienceData -DataTransfer,DataStructure -DataTransfer,DataGridManagementSystem -DataTransfer,DataManagementSystems -DataTransfer,Data volume -DataTransfer,DataSystemDevelopment -DataTransferFactory,DataTransferFactory -DataTransferFactory,Data nodes -DataTransferFactory,Data Transfer extension point -DataTransferFactory,Data products -DataTransferFactory,DataNodes -DataTransferFactory,Data registry -DataTransferFactory,CoreData -DataTransferFactory,DataSystemPDS4InformationModel-Driven -DataTransferFactory,DataList -DataTransferFactory,GoddardEarthScienceData -DataTransferFactory,DataStructure -DataTransferFactory,DataGridManagementSystem -DataTransferFactory,Data volume -InPlaceDataTransferer,InPlaceDataTransferer -InPlaceDataTransferFactory,InPlaceDataTransferFactory -LocalDataTransferer,LocalDataTransferer -LocalDataTransferFactory,LocalDataTransferFactory -LocalDataTransferFactory,LocalDataTransfer -RemoteDataTransferer,RemoteDataTransferer -RemoteDataTransferer,RemoteFileSystemProvider -RemoteDataTransferFactory,RemoteDataTransferFactory -RemoteDataTransferFactory,RemoteDataTransfer -S3DataTransferer,S3DataTransferer -S3DataTransfererFactory,S3DataTransfererFactory -TestInPlaceDataTransferer,TestInPlaceDataTransferer -TestLocalDataTransferer,TestLocalDataTransferer -TestS3DataTransferer,TestS3DataTransferer -TestS3DataTransfererFactory,TestS3DataTransfererFactory -TransferStatusTracker,TransferStatusTracker -Cli,Cli -TestFileManagerCli,TestFileManagerCli -UseMockClientCmdLineActionStore,UseMockClientCmdLineActionStore -UseMockClientCmdLineActionStore,UseGit -UseMockClientCmdLineActionStore,Use npm -UseMockClientCmdLineActionStore,ElementStore -AbstractDeleteProductCliAction,AbstractDeleteProductCliAction -AbstractGetProductCliAction,AbstractGetProductCliAction -AbstractQueryCliAction,AbstractQueryCliAction -AddProductTypeCliAction,AddProductTypeCliAction -DeleteProductByIdCliAction,DeleteProductByIdCliAction -DeleteProductByNameCliAction,DeleteProductByNameCliAction -DumpMetadataCliAction,DumpMetadataCliAction -FileManagerCliAction,FileManagerCliAction -FileManagerCliAction,FileManagerAug -FileManagerCliAction,File Browser dialogs -FileManagerCliAction,FileManagerServer -FileManagerCliAction,FileManagerUsage -FileManagerCliAction,File Browser plugin -FileManagerCliAction,FileManagerPage -FileManagerCliAction,FileManagerManage -FileManagerCliAction,File Manager architecture -FileManagerCliAction,File Manager file System Provider -FileManagerCliAction,File information -FileManagerCliAction,File Manager loads -FileManagerCliAction,File Manager services -GetCurrentTransferCliAction,GetCurrentTransferCliAction -GetCurrentTransfersCliAction,GetCurrentTransfersCliAction -GetFilePercentTransferredCliAction,GetFilePercentTransferredCliAction -GetFirstPageCliAction,GetFirstPageCliAction -GetLastPageCliAction,GetLastPageCliAction -GetNextPageCliAction,GetNextPageCliAction -GetNumProductsCliAction,GetNumProductsCliAction -GetPrevPageCliAction,GetPrevPageCliAction -GetProductByIdCliAction,GetProductByIdCliAction -GetProductByNameCliAction,GetProductByNameCliAction -GetProductPercentTransferredCliAction,GetProductPercentTransferredCliAction -GetProductTypeByNameCliAction,GetProductTypeByNameCliAction -HasProductCliAction,HasProductCliAction -IngestProductCliAction,IngestProductCliAction -LuceneQueryCliAction,LuceneQueryCliAction -LuceneQueryCliAction,LuceneIndexCatalog -LuceneQueryCliAction,default Apache Lucene -RetrieveFilesCliAction,RetrieveFilesCliAction -SqlQueryCliAction,SqlQueryCliAction -Util,Util -Pagination,Pagination -DbStructFactory,DbStructFactory -GenericFileManagerObjectFactory,GenericFileManagerObjectFactory -QueryUtils,QueryUtils -SqlParser,SqlParser -TestGenericFileManagerObjectStructFactory,TestGenericFileManagerObjectStructFactory -TestXmlRpcStructFactory,TestXmlRpcStructFactory -TestXmlStructFactory,TestXmlStructFactory -XmlRpcStructFactory,XmlRpcStructFactory -XmlStructFactory,XmlStructFactory -tools,tools -CASAnalyzer,CASAnalyzer -CatalogSearch,CatalogSearch -CatalogSearch,XMLMetadataConceptCatalog -CatalogSearch,LuceneIndexCatalog -CatalogSearch,AMGAMetadataCatalog -CatalogSearch,Catalog extension point interface -CatalogSearch,Catalog extension point -CatalogSearch,Catalog interface -CatalogSearch,ApacheOODTCatalog -DeleteProduct,DeleteProduct -DeleteProduct,ProductGenerationExecutives -DeleteProduct,ProductPctTransferred -DeleteProduct,ScienceProduct -DeleteProduct,Product instance information -DeleteProduct,Product instance metadata -DumpDbElementsToXml,DumpDbElementsToXml -ExpImpCatalog,ExpImpCatalog -MetadataBasedProductMover,MetadataBasedProductMover -MetadataBasedProductMover,ProductMetadata -MetadataBasedProductMover,Metadata objects -MetadataBasedProductMover,Metadata information -MetadataBasedProductMover,Metadata management -MetadataBasedProductMover,custom PGE Metadata -MetadataBasedProductMover,Metadata generation -MetadataDumper,MetadataDumper -Metadata,class FileAttributesMetKeys -Metadata,class ProductMetKeys -Metadata,class FilemgrMetExtractor -Metadata,class AbstractFilemgrMetExtractor -Metadata,class CoreMetExtractor -Metadata,class CoreMetKeys -OptimizeLuceneCatalog,OptimizeLuceneCatalog -OptimizeLuceneCatalog,XMLMetadataConceptCatalog -OptimizeLuceneCatalog,Catalog extension point interface -OptimizeLuceneCatalog,LuceneCatalog -ProductDumper,ProductDumper -ProductTypeDocTool,ProductTypeDocTool -ProductTypeDocTool,Product structures -ProductTypeDocTool,ProductMetadata -ProductTypeDocTool,ProductGenerationExecutives -ProductTypeDocTool,QueryTool -ProductTypeDocTool,ProductPctTransferred -ProductTypeDocTool,ProductId -ProductTypeDocTool,ProductTypes -ProductTypeDocTool,ScienceProduct -ProductTypeDocTool,ProductStructure -ProductTypeDocTool,Product operation -ProductTypeDocTool,Product instance information -ProductTypeDocTool,Product listing pages -QueryTool,QueryTool -RangeQueryTester,RangeQueryTester -SolrIndexer,SolrIndexer -CliAction,CliAction diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/A FRAMEWORK FOR COLLABORATIVE REVIEW OF CANDIDATE EVENTS IN HIGH DATA RATE STREAMS THE V-FASTR EXPERIMENT AS A CASE STUDY-ziyan.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/A FRAMEWORK FOR COLLABORATIVE REVIEW OF CANDIDATE EVENTS IN HIGH DATA RATE STREAMS THE V-FASTR EXPERIMENT AS A CASE STUDY-ziyan.txt deleted file mode 100644 index fa4a2dec90f51473e5122c2fa80c2ebeb85a5499..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/doc/Apache OODT File Manager/A FRAMEWORK FOR COLLABORATIVE REVIEW OF CANDIDATE EVENTS IN HIGH DATA RATE STREAMS THE V-FASTR EXPERIMENT AS A CASE STUDY-ziyan.txt +++ /dev/null @@ -1,694 +0,0 @@ -structs , structs -Boolean Query Criteria , Boolean Query Criteria -Element , Element -Extractor Spec , Extractor Spec -Extractor Spec , spicified Extractor extractors -Extractor Spec , Mime Type Extractor -Extractor Spec , Core Met Extractor -Extractor Spec , Job Spec -File Transfer Status , File Transfer Status -File Transfer Status , File Manager extension points -File Transfer Status , File Manager Dashboard Nulled -File Transfer Status , Dev Extreme Java Script File -File Transfer Status , File Manager UI component -File Transfer Status , File Manager server interface -File Transfer Status , File Manager Command class -File Transfer Status , File Manager web interface -File Transfer Status , File Manager App UI -File Transfer Status , File content -File Transfer Status , JSX File -File Transfer Status , File Manager file System Provider -File Transfer Status , File type -File Transfer Status , File information -Free Text Query Criteria , Free Text Query Criteria -Free Text Query Criteria , Free Pastry -Free Text Query Criteria , implementation Free -Product , Product -Product Page , Product Page -Product Page , Product structures -Product Page , Product Metadata -Product Page , Product Generation Executives -Product Page , Product Pct Transferred -Product Page , Product Id -Product Page , File Manager Page -Product Page , Product Types -Product Page , Product Structure -Product Page , Product operation -Product Page , Product instance information -Product Page , Product listing pages -Product Type , Product Type -Product Type , Product structures -Product Type , Product Metadata -Product Type , Type Script script -Product Type , Product Generation Executives -Product Type , Product Pct Transferred -Product Type , Product Id -Product Type , Product Types -Product Type , Product Structure -Product Type , Product operation -Product Type , Product instance information -Product Type , product Type -Product Type , Product listing pages -Query , Query -Query Criteria , Query Criteria -Range Query Criteria , Range Query Criteria -Reference , Reference -Term Query Criteria , Term Query Criteria -Test Product , Test Product -Test Reference , Test Reference -System , System -Xml Rpc File Manager , Xml Rpc File Manager -Xml Rpc File Manager , import File Manager -Xml Rpc File Manager , Open File Manager -Xml Rpc File Manager , Xml Rpc File Manager Client -Xml Rpc File Manager , Trillo File Manager -Xml Rpc File Manager , Kendo UI File Manager -Xml Rpc File Manager , Manager The Repository Manager extension -Xml Rpc File Manager , Extreme Java Script File Manager -Xml Rpc File Manager , Repository Manager The Repository Manager -Xml Rpc File Manager , CAS File Manager -Xml Rpc File Manager , Xml Rpc File Manager complex -Xml Rpc File Manager , Manager extension point -Xml Rpc File Manager , Webix File Manager -Xml Rpc File Manager , bs4 File Manager -Xml Rpc File Manager , Apache OODT File Manager -Xml Rpc File Manager Client , Xml Rpc File Manager Client -Xml Rpc File Manager Client , Client implementation -Xml Rpc File Manager Client , Client React connector -Xml Rpc File Manager Client , Client React component -Xml Rpc File Manager Client , Client side metadata -Xml Rpc File Manager Client , Mahasen Client -Dispatcher , Dispatcher -Result , Result -Secure Web Server , Secure Web Server -Secure Web Server , Apache HTTPD Web Server -Secure Web Server , Server Node API v1 Localization -Secure Web Server , Windows Server -Repository , Repository -Repository Manager , Repository Manager -Repository Manager , Software Package Manager -Repository Manager , Task Manager -Repository Manager , import File Manager -Repository Manager , Open File Manager -Repository Manager , NET Core File Manager -Repository Manager , OODT CAS File Manager -Repository Manager , Node Package Manager -Repository Manager , Trillo File Manager -Repository Manager , Kendo UI File Manager -Repository Manager , Word Press File Manager -Repository Manager , Manager The Repository Manager extension -Repository Manager , Manager Ingest Use Case -Repository Manager , Extreme Java Script File Manager -Repository Manager , Repository Manager extension point -Repository Manager , Repository Manager The Repository Manager -Repository Manager , Repository Manager The Repository Manager -Repository Manager , CAS File Manager -Repository Manager , Manager Object Model -Repository Manager , demand Library Manager -Repository Manager , Deploy Trillo File Manager -Repository Manager , Manager extension point -Repository Manager , Webix File Manager -Repository Manager , bs4 File Manager -Repository Manager , Apache OODT File Manager -Repository Manager Factory , Repository Manager Factory -Repository Manager Factory , Repository Manager extension point -Repository Manager Factory , Repository Manager The Repository Manager -Data Source Repository Manager , Data Source Repository Manager -Data Source Repository Manager , Software Package Manager -Data Source Repository Manager , Data nodes -Data Source Repository Manager , Manager control -Data Source Repository Manager , Task Manager -Data Source Repository Manager , import File Manager -Data Source Repository Manager , Open File Manager -Data Source Repository Manager , NET Core File Manager -Data Source Repository Manager , OODT CAS File Manager -Data Source Repository Manager , Node Package Manager -Data Source Repository Manager , Moneydance Financial Data -Data Source Repository Manager , Data Transfer interface -Data Source Repository Manager , Manager The Repository Manager extension -Data Source Repository Manager , Data products -Data Source Repository Manager , Extreme Java Script File Manager -Data Source Repository Manager , Manager Architecture -Data Source Repository Manager , Data registry -Data Source Repository Manager , Manager catalog -Data Source Repository Manager , Repository Manager The Repository Manager -Data Source Repository Manager , CAS File Manager -Data Source Repository Manager , Manager Object Model -Data Source Repository Manager , demand Library Manager -Data Source Repository Manager , Manager extension point -Data Source Repository Manager , Webix File Manager -Data Source Repository Manager , Data Management Systems -Data Source Repository Manager , Data volume -Data Source Repository Manager , Data System Development -Data Source Repository Manager Factory , Data Source Repository Manager Factory -Data Source Repository Manager Factory , Data nodes -Data Source Repository Manager Factory , Moneydance Financial Data -Data Source Repository Manager Factory , Data Transfer extension point -Data Source Repository Manager Factory , Data Transfer interface -Data Source Repository Manager Factory , Data products -Data Source Repository Manager Factory , Data registry -Data Source Repository Manager Factory , Data Grid Management System -Data Source Repository Manager Factory , Data Management Systems -Data Source Repository Manager Factory , Data volume -Data Source Repository Manager Factory , Data System Development -Science Data Repository Manager , Science Data Repository Manager -Science Data Repository Manager , Software Package Manager -Science Data Repository Manager , Manager control -Science Data Repository Manager , Task Manager -Science Data Repository Manager , import File Manager -Science Data Repository Manager , Open File Manager -Science Data Repository Manager , NET Core File Manager -Science Data Repository Manager , OODT CAS File Manager -Science Data Repository Manager , Science Pge Config File Writer -Science Data Repository Manager , Node Package Manager -Science Data Repository Manager , Trillo File Manager -Science Data Repository Manager , Kendo UI File Manager -Science Data Repository Manager , Word Press File Manager -Science Data Repository Manager , Manager The Repository Manager extension -Science Data Repository Manager , Computer Science -Science Data Repository Manager , Extreme Java Script File Manager -Science Data Repository Manager , Manager Architecture -Science Data Repository Manager , Science Product -Science Data Repository Manager , Manager catalog -Science Data Repository Manager , Repository Manager The Repository Manager -Science Data Repository Manager , CAS File Manager -Science Data Repository Manager , Manager Object Model -Science Data Repository Manager , Space Science -Science Data Repository Manager , demand Library Manager -Science Data Repository Manager , Deploy Trillo File Manager -Science Data Repository Manager , Manager extension point -Science Data Repository Manager , Webix File Manager -Science Data Repository Manager , bs4 File Manager -Science Data Repository Manager , Apache OODT File Manager -Science Data Repository Manager Factory , Science Data Repository Manager Factory -Science Data Repository Manager Factory , Computer Science -Science Data Repository Manager Factory , Science Product -Science Data Repository Manager Factory , Space Science -Science Data Repository Manager Factory , Science Data Processing Systems -Test XML Repository Manager , Test XML Repository Manager -Test XML Repository Manager , Software Package Manager -Test XML Repository Manager , Manager control -Test XML Repository Manager , Task Manager -Test XML Repository Manager , import File Manager -Test XML Repository Manager , Open File Manager -Test XML Repository Manager , Node Package Manager -Test XML Repository Manager , Trillo File Manager -Test XML Repository Manager , Manager The Repository Manager extension -Test XML Repository Manager , Extreme Java Script File Manager -Test XML Repository Manager , Manager Architecture -Test XML Repository Manager , Manager catalog -Test XML Repository Manager , Repository Manager The Repository Manager -Test XML Repository Manager , CAS File Manager -Test XML Repository Manager , Manager Object Model -Test XML Repository Manager , demand Library Manager -Test XML Repository Manager , Manager extension point -Test XML Repository Manager , Webix File Manager -Test XML Repository Manager , bs4 File Manager -XML Repository Manager , XML Repository Manager -XML Repository Manager , XML Metadata Concept Catalog -XML Repository Manager , Manager control -XML Repository Manager , Task Manager -XML Repository Manager , import File Manager -XML Repository Manager , Open File Manager -XML Repository Manager , NET Core File Manager -XML Repository Manager , OODT CAS File Manager -XML Repository Manager , XML files -XML Repository Manager , Trillo File Manager -XML Repository Manager , Kendo UI File Manager -XML Repository Manager , Word Press File Manager -XML Repository Manager , Manager The Repository Manager extension -XML Repository Manager , Manager Ingest Use Case -XML Repository Manager , Extreme Java Script File Manager -XML Repository Manager , Manager Architecture -XML Repository Manager , Manager catalog -XML Repository Manager , Repository Manager The Repository Manager -XML Repository Manager , CAS File Manager -XML Repository Manager , Deploy Trillo File Manager -XML Repository Manager , Webix File Manager -XML Repository Manager , bs4 File Manager -XML Repository Manager , Apache OODT File Manager -XML Repository Manager Factory , XML Repository Manager Factory -XML Repository Manager Factory , XML files -Metadata , Metadata -Core Met Keys , Core Met Keys -Core Met Keys , Core machines -Core Met Keys , Core Data -File Attributes Met Keys , File Attributes Met Keys -File Attributes Met Keys , File Managers functionality -File Attributes Met Keys , File Browser dialogs -File Attributes Met Keys , File Retrieval System -File Attributes Met Keys , File Upload Component -File Attributes Met Keys , Dev Extreme Java Script File -File Attributes Met Keys , File Location Extractor -File Attributes Met Keys , File Pct Transferred -File Attributes Met Keys , File Repository layouts -File Attributes Met Keys , File content -File Attributes Met Keys , JSX File -File Attributes Met Keys , File type -Product Met Keys , Product Met Keys -Product Met Keys , Product structures -Product Met Keys , Product Metadata -Product Met Keys , Product Id -Product Met Keys , Product Types -Product Met Keys , Science Product -Product Met Keys , Product Structure -Product Met Keys , Product operation -Filemgr Met Extractor , Filemgr Met Extractor -Filemgr Met Extractor , Core Met Extractor -Abstract Filemgr Met Extractor , Abstract Filemgr Met Extractor -Abstract Filemgr Met Extractor , Mime Type Extractor -Abstract Filemgr Met Extractor , Core Met Extractor -Core Met Extractor , Core Met Extractor -Core Met Extractor , Core machines -Core Met Extractor , Extractor extractors -Core Met Extractor , Mime Type Extractor -Core Met Extractor , Core Data -Test Abstract Filemgr Met Extractor , Test Abstract Filemgr Met Extractor -Test Abstract Filemgr Met Extractor , Extractor extractors -Test Abstract Filemgr Met Extractor , Mime Type Extractor -Test Abstract Filemgr Met Extractor , Core Met Extractor -Test Core Met Extractor , Test Core Met Extractor -Test Core Met Extractor , Extractor extractors -Test Core Met Extractor , Mime Type Extractor -Test Core Met Extractor , Core Met Extractor -Catalog , Catalog -Catalog Factory , Catalog Factory -Data Source Catalog , Data Source Catalog -Data Source Catalog , XML Metadata Concept Catalog -Data Source Catalog , Data nodes -Data Source Catalog , Lucene Index Catalog -Data Source Catalog , Catalog extension point interface -Data Source Catalog , Data Transfer extension point -Data Source Catalog , Data products -Data Source Catalog , Data Nodes -Data Source Catalog , Data registry -Data Source Catalog , Core Data -Data Source Catalog , Data System PDS4 Information Model-Driven -Data Source Catalog , Lucene Catalog -Data Source Catalog , Data List -Data Source Catalog , Catalog interface -Data Source Catalog , Goddard Earth Science Data -Data Source Catalog , Data Structure -Data Source Catalog , Data Grid Management System -Data Source Catalog , Data volume -Data Source Catalog Factory , Data Source Catalog Factory -Data Source Catalog Factory , Data nodes -Data Source Catalog Factory , Data Intensive Scientific -Data Source Catalog Factory , Moneydance Financial Data -Data Source Catalog Factory , Data Transfer interface -Data Source Catalog Factory , Data products -Data Source Catalog Factory , Data Nodes -Data Source Catalog Factory , Data registry -Data Source Catalog Factory , Core Data -Data Source Catalog Factory , Data System PDS4 Information Model-Driven -Data Source Catalog Factory , Data List -Data Source Catalog Factory , Data Structure -Data Source Catalog Factory , Data Management Systems -Data Source Catalog Factory , Data volume -Data Source Catalog Factory , Data System Development -Lenient Data Source Catalog , Lenient Data Source Catalog -Lenient Data Source Catalog , Lucene Index Catalog -Lenient Data Source Catalog , AMGA Metadata Catalog -Lenient Data Source Catalog , Catalog extension point -Lenient Data Source Catalog , Lucene Catalog -Lenient Data Source Catalog , Catalog interface -Lenient Data Source Catalog , Apache OODT Catalog -Lucene Catalog , Lucene Catalog -Lucene Catalog , XML Metadata Concept Catalog -Lucene Catalog , Lucene Index Catalog -Lucene Catalog , Lucene Index Catalog -Lucene Catalog , AMGA Metadata Catalog -Lucene Catalog , Catalog extension point interface -Lucene Catalog , Catalog extension point -Lucene Catalog , Apache OODT Catalog -Lucene Catalog , default Apache Lucene -Lucene Catalog Factory , Lucene Catalog Factory -Lucene Catalog Factory , Lucene Catalog -Mapped Data Source Catalog , Mapped Data Source Catalog -Mapped Data Source Catalog , Lucene Index Catalog -Mapped Data Source Catalog , AMGA Metadata Catalog -Mapped Data Source Catalog , Catalog extension point -Mapped Data Source Catalog , Lucene Catalog -Mapped Data Source Catalog , Catalog interface -Mapped Data Source Catalog , Apache OODT Catalog -Mapped Data Source Catalog Factory , Mapped Data Source Catalog Factory -Mock Catalog , Mock Catalog -Mock Catalog Factory , Mock Catalog Factory -Science Data Catalog , Science Data Catalog -Science Data Catalog , XML Metadata Concept Catalog -Science Data Catalog , AMGA Metadata Catalog -Science Data Catalog , Science Pge Config File Writer -Science Data Catalog , Catalog extension point interface -Science Data Catalog , Computer Science -Science Data Catalog , Science Product -Science Data Catalog , Lucene Catalog -Science Data Catalog , Catalog interface -Science Data Catalog , Apache OODT Catalog -Science Data Catalog , Space Science -Science Data Catalog , Science Data Processing Systems -Science Data Catalog Factory , Science Data Catalog Factory -Science Data Catalog Factory , Science Pge Config File Writer -Science Data Catalog Factory , Computer Science -Science Data Catalog Factory , Science Product -Science Data Catalog Factory , Space Science -Catalog , Catalog -Product Id Generator , Product Id Generator -Product Id Generator , Product structures -Product Id Generator , Product Id -Product Id Generator , Product Types -Product Id Generator , Science Product -Product Id Generator , Product Structure -Product Id Generator , Product operation -Product Serializer , Product Serializer -Complete Product , Complete Product -Complete Product , Product Generation Executives -Complete Product , Product Pct Transferred -Complete Product , Science Product -Complete Product , Product instance information -Complete Product , Product listing pages -Default Product Serializer , Default Product Serializer -Default Product Serializer , Default configuration -Name Product Id Generator , Name Product Id Generator -Name Product Id Generator , Name Node -Parameters , Parameters -Query Response , Query Response -Solr Catalog , Solr Catalog -Solr Catalog Factory , Solr Catalog Factory -Solr Client , Solr Client -UUID Product Id Generator , UUID Product Id Generator -Validation , Validation -Validation Layer , Validation Layer -Validation Layer , Validation Layer The Validation Layer -Validation Layer , Validation Layer The Validation Layer -Validation Layer , Validation Layer extension point -Validation Layer , Layer extension point -Validation Layer , Layer The Validation Layer extension -Validation Layer Factory , Validation Layer Factory -Validation Layer Factory , Validation Layer The Validation Layer -Validation Layer Factory , Validation Layer extension point -Data Source Validation Layer , Data Source Validation Layer -Data Source Validation Layer , Data nodes -Data Source Validation Layer , Data Intensive Scientific -Data Source Validation Layer , Data View -Data Source Validation Layer , Moneydance Financial Data -Data Source Validation Layer , Validation Layer The Validation Layer -Data Source Validation Layer , Data products -Data Source Validation Layer , Data Nodes -Data Source Validation Layer , Data registry -Data Source Validation Layer , Core Data -Data Source Validation Layer , Data System PDS4 Information Model-Driven -Data Source Validation Layer , Data List -Data Source Validation Layer , Data Source -Data Source Validation Layer , Data Protocols -Data Source Validation Layer , Layer extension point -Data Source Validation Layer , Data Structure -Data Source Validation Layer , Layer The Validation Layer extension -Data Source Validation Layer , Data Grid Management System -Data Source Validation Layer , Data Management Systems -Data Source Validation Layer , Data volume -Data Source Validation Layer , Data System Development -Data Source Validation Layer Factory , Data Source Validation Layer Factory -Data Source Validation Layer Factory , Data nodes -Data Source Validation Layer Factory , Data Intensive Scientific -Data Source Validation Layer Factory , Data View -Data Source Validation Layer Factory , Moneydance Financial Data -Data Source Validation Layer Factory , Data Transfer extension point -Data Source Validation Layer Factory , Data products -Data Source Validation Layer Factory , Data Nodes -Data Source Validation Layer Factory , Data registry -Data Source Validation Layer Factory , Core Data -Data Source Validation Layer Factory , Data List -Data Source Validation Layer Factory , Goddard Earth Science Data -Data Source Validation Layer Factory , Data Source -Data Source Validation Layer Factory , Data Protocols -Data Source Validation Layer Factory , Data Structure -Data Source Validation Layer Factory , Data Grid Management System -Data Source Validation Layer Factory , Data Management Systems -Data Source Validation Layer Factory , Data volume -Data Source Validation Layer Factory , Data System Development -Science Data Validation Layer , Science Data Validation Layer -Science Data Validation Layer , Science Pge Config File Writer -Science Data Validation Layer , Validation Layer The Validation Layer -Science Data Validation Layer , Computer Science -Science Data Validation Layer , Science Product -Science Data Validation Layer , Game Science -Science Data Validation Layer , Space Science -Science Data Validation Layer , Layer extension point -Science Data Validation Layer , Science Data Processing Systems -Science Data Validation Layer , Layer The Validation Layer extension -Science Data Validation Layer Factory , Science Data Validation Layer Factory -Science Data Validation Layer Factory , Computer Science -Science Data Validation Layer Factory , Science Product -Science Data Validation Layer Factory , Game Science -Science Data Validation Layer Factory , Space Science -Science Data Validation Layer Factory , Science Data Processing Systems -Test XML Validation Layer , Test XML Validation Layer -Test XML Validation Layer , Validation Layer The Validation Layer -Test XML Validation Layer , Layer extension point -Test XML Validation Layer , Layer The Validation Layer extension -XML Validation Layer , XML Validation Layer -XML Validation Layer , XML Metadata Concept Catalog -XML Validation Layer , XML files -XML Validation Layer , Validation Layer The Validation Layer -XML Validation Layer , XML Metadata Concept catalog -XML Validation Layer , XML syntax -XML Validation Layer , Layer The Validation Layer extension -XML Validation Layer Factory , XML Validation Layer Factory -XML Validation Layer Factory , XML files -XML Validation Layer Factory , XML syntax -versioning , versioning -Versioner , Versioner -Versioner , class Acquisition Date Versioner -Versioner , class Basic Versioner -Versioner , class Configurable Metadata Based File Versioner -Versioner , class Date Time Versioner -Versioner , class Directory Product Versioner -Versioner , class In Place Versioner -Versioner , class Metadata Based File Versioner -Versioner , class Product Type Met Versioner -Versioner , class Single File Basic Versioner -Acquisition Date Versioner , Acquisition Date Versioner -Basic Versioner , Basic Versioner -Basic Versioner , Versioner extension point -Configurable Metadata Based File Versioner , Configurable Metadata Based File Versioner -Configurable Metadata Based File Versioner , Versioner extension point -Date Time Versioner , Date Time Versioner -Directory Product Versioner , Directory Product Versioner -In Place Versioner , In Place Versioner -Metadata Based File Versioner , Metadata Based File Versioner -Metadata Based File Versioner , Versioner extension point -Metadata Based File Versioner , Product Metadata -Metadata Based File Versioner , Metadata objects -Metadata Based File Versioner , Metadata information -Metadata Based File Versioner , Metadata management -Product Type Met Versioner , Product Type Met Versioner -Product Type Met Versioner , Versioner extension point -Product Type Met Versioner , Product structures -Product Type Met Versioner , Product Metadata -Product Type Met Versioner , Product Generation Executives -Product Type Met Versioner , Product Pct Transferred -Product Type Met Versioner , Product Id -Product Type Met Versioner , Product Types -Product Type Met Versioner , Science Product -Product Type Met Versioner , Product Structure -Product Type Met Versioner , Product operation -Product Type Met Versioner , Product instance information -Product Type Met Versioner , Product listing pages -Single File Basic Versioner , Single File Basic Versioner -Single File Basic Versioner , Versioner extension point -Test Acquisition Date Versioner , Test Acquisition Date Versioner -Test Basic Versioner , Test Basic Versioner -Test Configurable Metadata Based File Versioner , Test Configurable Metadata Based File Versioner -Test Configurable Metadata Based File Versioner , Versioner extension point -Test Date Time Versioner , Test Date Time Versioner -Test Directory Based Product Versioner , Test Directory Based Product Versioner -Test Directory Based Product Versioner , Versioner extension point -Test In Place Versioner , Test In Place Versioner -Test In Place Versioner , Versioner extension point -Test Metadata Based File Versioner , Test Metadata Based File Versioner -Test Metadata Based File Versioner , Versioner extension point -Test Product Type Met Versioner , Test Product Type Met Versioner -Test Product Type Met Versioner , Versioner extension point -Test Single File Basic Versioner , Test Single File Basic Versioner -Test Single File Basic Versioner , Versioner extension point -Versioning Utils , Versioning Utils -ingest , ingest -Cache , Cache -Cache Factory , Cache Factory -Ingester , Ingester -Remoteable Cache , Remoteable Cache -Abstract Cache Server Factory , Abstract Cache Server Factory -Cached Ingester , Cached Ingester -CmdLine Ingester , CmdLine Ingester -Local Cache , Local Cache -Local Cache Factory , Local Cache Factory -Rmi Cache , Rmi Cache -Rmi Cache Factory , Rmi Cache Factory -Rmi Cache Server , Rmi Cache Server -Rmi Cache Server Factory , Rmi Cache Server Factory -Std Ingester , Std Ingester -Test Cached Ingester , Test Cached Ingester -Test Local Cache , Test Local Cache -Test Rmi Cache , Test Rmi Cache -Test Std Ingester , Test Std Ingester -datatransfer , datatransfer -Data Transfer , Data Transfer -Data Transfer , Data nodes -Data Transfer , Data Intensive Scientific -Data Transfer , Local Data Transfer -Data Transfer , client Transfer -Data Transfer , Moneydance Financial Data -Data Transfer , Place Data Transfer -Data Transfer , Data Transfer extension point -Data Transfer , Data Transfer interface -Data Transfer , Data products -Data Transfer , Data Nodes -Data Transfer , Remote Data Transfer -Data Transfer , Data registry -Data Transfer , Data System PDS4 Information Model-Driven -Data Transfer , Data List -Data Transfer , Goddard Earth Science Data -Data Transfer , Data Structure -Data Transfer , Data Grid Management System -Data Transfer , Data Management Systems -Data Transfer , Data volume -Data Transfer , Data System Development -Data Transfer Factory , Data Transfer Factory -Data Transfer Factory , Data nodes -Data Transfer Factory , Data Transfer extension point -Data Transfer Factory , Data products -Data Transfer Factory , Data Nodes -Data Transfer Factory , Data registry -Data Transfer Factory , Core Data -Data Transfer Factory , Data System PDS4 Information Model-Driven -Data Transfer Factory , Data List -Data Transfer Factory , Goddard Earth Science Data -Data Transfer Factory , Data Structure -Data Transfer Factory , Data Grid Management System -Data Transfer Factory , Data volume -In Place Data Transferer , In Place Data Transferer -In Place Data Transfer Factory , In Place Data Transfer Factory -Local Data Transferer , Local Data Transferer -Local Data Transfer Factory , Local Data Transfer Factory -Local Data Transfer Factory , Local Data Transfer -Remote Data Transferer , Remote Data Transferer -Remote Data Transferer , Remote File System Provider -Remote Data Transfer Factory , Remote Data Transfer Factory -Remote Data Transfer Factory , Remote Data Transfer -S3Data Transferer , S3Data Transferer -S3Data Transferer Factory , S3Data Transferer Factory -Test In Place Data Transferer , Test In Place Data Transferer -Test Local Data Transferer , Test Local Data Transferer -Test S3 Data Transferer , Test S3 Data Transferer -Test S3 Data Transferer Factory , Test S3 Data Transferer Factory -Transfer Status Tracker , Transfer Status Tracker -Cli , Cli -Test File Manager Cli , Test File Manager Cli -Use Mock Client Cmd Line Action Store , Use Mock Client Cmd Line Action Store -Use Mock Client Cmd Line Action Store , Use Git -Use Mock Client Cmd Line Action Store , Use npm -Use Mock Client Cmd Line Action Store , Element Store -Abstract Delete Product Cli Action , Abstract Delete Product Cli Action -Abstract Get Product Cli Action , Abstract Get Product Cli Action -Abstract Query Cli Action , Abstract Query Cli Action -Add Product Type Cli Action , Add Product Type Cli Action -Delete Product By Id Cli Action , Delete Product By Id Cli Action -Delete Product By Name Cli Action , Delete Product By Name Cli Action -Dump Metadata Cli Action , Dump Metadata Cli Action -File Manager Cli Action , File Manager Cli Action -File Manager Cli Action , File Manager Aug -File Manager Cli Action , File Browser dialogs -File Manager Cli Action , File Manager Server -File Manager Cli Action , File Manager Usage -File Manager Cli Action , File Browser plugin -File Manager Cli Action , File Manager Page -File Manager Cli Action , File Manager Manage -File Manager Cli Action , File Manager architecture -File Manager Cli Action , File Manager file System Provider -File Manager Cli Action , File information -File Manager Cli Action , File Manager loads -File Manager Cli Action , File Manager services -Get Current Transfer Cli Action , Get Current Transfer Cli Action -Get Current Transfers Cli Action , Get Current Transfers Cli Action -Get File Percent Transferred Cli Action , Get File Percent Transferred Cli Action -Get First Page Cli Action , Get First Page Cli Action -Get Last Page Cli Action , Get Last Page Cli Action -Get Next Page Cli Action , Get Next Page Cli Action -Get Num Products Cli Action , Get Num Products Cli Action -Get Prev Page Cli Action , Get Prev Page Cli Action -Get Product By Id Cli Action , Get Product By Id Cli Action -Get Product By Name Cli Action , Get Product By Name Cli Action -Get Product Percent Transferred Cli Action , Get Product Percent Transferred Cli Action -Get Product Type By Name Cli Action , Get Product Type By Name Cli Action -Has Product Cli Action , Has Product Cli Action -Ingest Product Cli Action , Ingest Product Cli Action -Lucene Query Cli Action , Lucene Query Cli Action -Lucene Query Cli Action , Lucene Index Catalog -Lucene Query Cli Action , default Apache Lucene -Retrieve Files Cli Action , Retrieve Files Cli Action -Sql Query Cli Action , Sql Query Cli Action -Util , Util -Pagination , Pagination -Db Struct Factory , Db Struct Factory -Generic File Manager Object Factory , Generic File Manager Object Factory -Query Utils , Query Utils -Sql Parser , Sql Parser -Test Generic File Manager Object Struct Factory , Test Generic File Manager Object Struct Factory -Test Xml Rpc Struct Factory , Test Xml Rpc Struct Factory -Test Xml Struct Factory , Test Xml Struct Factory -Xml Rpc Struct Factory , Xml Rpc Struct Factory -Xml Struct Factory , Xml Struct Factory -tools , tools -CAS Analyzer , CAS Analyzer -Catalog Search , Catalog Search -Catalog Search , XML Metadata Concept Catalog -Catalog Search , Lucene Index Catalog -Catalog Search , AMGA Metadata Catalog -Catalog Search , Catalog extension point interface -Catalog Search , Catalog extension point -Catalog Search , Catalog interface -Catalog Search , Apache OODT Catalog -Delete Product , Delete Product -Delete Product , Product Generation Executives -Delete Product , Product Pct Transferred -Delete Product , Science Product -Delete Product , Product instance information -Delete Product , Product instance metadata -Dump Db Elements To Xml , Dump Db Elements To Xml -ExpImp Catalog , ExpImp Catalog -Metadata Based Product Mover , Metadata Based Product Mover -Metadata Based Product Mover , Product Metadata -Metadata Based Product Mover , Metadata objects -Metadata Based Product Mover , Metadata information -Metadata Based Product Mover , Metadata management -Metadata Based Product Mover , custom PGE Metadata -Metadata Based Product Mover , Metadata generation -Metadata Dumper , Metadata Dumper -Metadata , class File Attributes Met Keys -Metadata , class Product Met Keys -Metadata , class Filemgr Met Extractor -Metadata , class Abstract Filemgr Met Extractor -Metadata , class Core Met Extractor -Metadata , class Core Met Keys -Optimize Lucene Catalog , Optimize Lucene Catalog -Optimize Lucene Catalog , XML Metadata Concept Catalog -Optimize Lucene Catalog , Catalog extension point interface -Optimize Lucene Catalog , Lucene Catalog -Product Dumper , Product Dumper -Product Type Doc Tool , Product Type Doc Tool -Product Type Doc Tool , Product structures -Product Type Doc Tool , Product Metadata -Product Type Doc Tool , Product Generation Executives -Product Type Doc Tool , Query Tool -Product Type Doc Tool , Product Pct Transferred -Product Type Doc Tool , Product Id -Product Type Doc Tool , Product Types -Product Type Doc Tool , Science Product -Product Type Doc Tool , Product Structure -Product Type Doc Tool , Product operation -Product Type Doc Tool , Product instance information -Product Type Doc Tool , Product listing pages -Query Tool , Query Tool -Range Query Tester , Range Query Tester -Solr Indexer , Solr Indexer -Cli Action , Cli Action \ No newline at end of file diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/A FRAMEWORK FOR COLLABORATIVE REVIEW OF CANDIDATE EVENTS IN HIGH DATA RATE STREAMS THE V-FASTR EXPERIMENT AS A CASE STUDY.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/A FRAMEWORK FOR COLLABORATIVE REVIEW OF CANDIDATE EVENTS IN HIGH DATA RATE STREAMS THE V-FASTR EXPERIMENT AS A CASE STUDY.txt deleted file mode 100644 index c92ae4773b74b21b9aa7b209b085a9f0f86cb88b..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/doc/Apache OODT File Manager/A FRAMEWORK FOR COLLABORATIVE REVIEW OF CANDIDATE EVENTS IN HIGH DATA RATE STREAMS THE V-FASTR EXPERIMENT AS A CASE STUDY.txt +++ /dev/null @@ -1,637 +0,0 @@ -A FRAMEWORK FOR COLLABORATIVE REVIEW OF CANDIDATE EVENTS IN HIGH DATA RATE -STREAMS: THE V-FASTR EXPERIMENT AS A CASE STUDY -Andrew F. Hart, Luca Cinquini, Shakeh E. Khudikyan, David R. Thompson, -Chris A. Mattmann, Kiri Wagstaff, Joseph Lazio, and Dayton Jones -Jet Propulsion Laboratory, California Institute of Technology, Pasadena, CA 91109, USA; andrew.f.hart@jpl.nasa.gov -Received 2014 March 24; accepted 2014 August 10; published 2014 December 16 -ABSTRACT -“Fast radio transients” are defined here as bright millisecond pulses of radio-frequency energy. These shortduration -pulses can be produced by known objects such as pulsars or potentially by more exotic objects such as -evaporating black holes. The identification and verification of such an event would be of great scientific value. This -is one major goal of the Very Long Baseline Array (VLBA) Fast Transient Experiment (V-FASTR), a softwarebased -detection system installed at the VLBA. V-FASTR uses a “commensal” (piggy-back) approach, analyzing -all array data continually during routine VLBA observations and identifying candidate fast transient events. Raw -data can be stored from a buffer memory, which enables a comprehensive off-line analysis. This is invaluable for -validating the astrophysical origin of any detection. Candidates discovered by the automatic system must be -reviewed each day by analysts to identify any promising signals that warrant a more in-depth investigation. To -support the timely analysis of fast transient detection candidates by V-FASTR scientists, we have developed a -metadata-driven, collaborative candidate review framework. The framework consists of a software pipeline for -metadata processing composed of both open source software components and project-specific code written -expressly to extract and catalog metadata from the incoming V-FASTR data products, and a web-based data portal -that facilitates browsing and inspection of the available metadata for candidate events extracted from the VLBA -radio data. -Key words: catalogs – methods: data analysis – pulsars: general – radio continuum: general -1. INTRODUCTION -One of the current primary goals of radio astronomy is to -explore and understand the “dynamic radio sky” (Cordes -et al. 2004). In contrast to generating catalogs of known -sources, this scientific thrust focuses on transient events, or -transient signals generated by persistent yet time-varying -sources. We do not yet fully understand the scope and -distribution of different transient sources, which range from -the known (e.g., active galactic nuclei, brown dwarfs, flare -stars, X-ray binaries, supernovae, gamma-ray bursts) to the -probable (e.g., exoplanets), to the possible (e.g., ET -civilizations, annihilating black holes). As noted by Cordes -et al. (2004, p.14), “most exciting would be the discovery of -new classes of sources” (italics in original). Radio telescopes -continue to increase their data collecting abilities, observing -the sky with progressively finer time resolution. Of current -particular interest is the detection and characterization of -“fast radio transients,” which last for only small fractions of a -second. -The V-FASTR experiment (Wayth et al. 2011) is one of a -new breed of radio astronomy experiments specifically -targeting fast transient radio signals. The experiment is -conducted in a fully commensal (passive) fashion, searching -for signals in the data gathered during the regular processing -activities of its host instrument. Unlike more traditional, -single-telescope observations, however, the V-FASTR -experiment simultaneously utilizes anywhere between 2 and -10 telescopes of the National Radio Astronomy Observatory -ʼs (NRAO) Very Long Baseline Array (VLBA) (Romney -2010). The VLBA consists of 10 25 m telescopes that are -positioned geographically such that no 2 are within each -otherʼs local horizon, and the V-FASTR experiment -leverages this configuration to better discriminate between -instances of terrestrial Radio Frequency Interference (RFI) -and potentially genuine astronomical pulses (Thompson -et al. 2011). -The huge volumes of raw time-series voltage data generated -by the VLBA in the course of its operation make storing the -full record of an entire observing session infeasible at the -present time. As a consequence, considerable effort has been -devoted to developing and fine-tuning algorithms for the realtime -identification of potentially interesting signals in the noisy -and often incomplete data (Thompson et al. 2011; Wayth et al. -2012). All data selected by the real-time processing step is -subsequently inspected, on a daily basis, by members of the -geographically distributed V-FASTR science team and either -discarded as spurious or archived offline for full analysis at a -later date. -The V-FASTR experiment must therefore operate within -several important resource constraints: the inability to archive -the full observational record due to space constraints, and a -practical workload constraint upon the human analysts -reviewing candidate detections. To address the latter, we have -developed a metadata-driven, collaborative candidate review -framework for the V-FASTR experiment. The framework -comprises a set of software components dedicated to the -automatic capture and organization of metadata describing the -candidate events identified as interesting by the automated -algorithms, and an online environment for the collaborative -perusal and inspection of related imagery data by the V-FASTR -analysis team. -The rest of this paper describes the system as follows. In -Section 2 we describe our project in a more general context. -Section 3 presents the methodology and an architectural -description of the system. We follow with an evaluation of -The Astronomical Journal, 149:23 (7pp), 2015 January doi:10.1088/0004-6256/149/1/23 -© 2015. The American Astronomical Society. All rights reserved. -1 -our experience deploying the framework in Section 4, and -offer conclusions and future directions for the work in -Section 5. -2. BACKGROUND -To better understand the context of the system implementation -presented in Section 3, we first briefly introduce the VFASTR -experiment and describe the development of scientific -data systems at the NASA Jet Propulsion Laboratory (JPL). -We then describe the Object Oriented Data Technology -(OODT) project, an open source information integration -platform that plays a central role in our framework. Finally, -we briefly touch upon several related efforts at developing -online tools to collaboratively classify and validate scientific -observations. -2.1. V-FASTR: The VLBA Fast TRansients Experiment -V-FASTR (VLBA Fast TRansients) is a data analysis -system used by the VLBA to detect candidate fast transient -events. Principal investigators submit observing proposals to -the VLBA targeted at galaxies, supernovae, quasars, pulsars, -and more. V-FASTR analyzes all data collected by the VLBA -as part of routine processing and produces a nightly list of -candidates identified within the data processed that day. The -raw data for each candidate is temporarily saved in case it is -needed to interpret or follow up on a particularly promising or -unusual detection. However, the raw data consumes significant -disk space and therefore the candidate list must be reviewed on -a timely basis by experts. False positives can be deleted and -their disk space reclaimed, while truly interesting events can be -re-processed to enable the generation of a sky image to localize -the source of the signal. Software tools that streamline and -simplify this review process are therefore highly valued by -candidate reviewers and can have a positive impact on other -similar efforts throughout the world. -2.2. Data System Development at JPL -The Data Management Systems and Technologies group at -the JPL develops software ground data systems to support -NASA science missions. These pipelines are specifically -optimized to support the data-intensive and computationallyintensive -processing steps often needed to convert raw remotesensing -observations into higher level data products at scale so -that they can be interpreted by the scientists. The process -almost always involves developing a close collaboration with -project scientists to obtain an understanding of the processing -algorithms involved, a sense of the scale and throughput -requirements, and other operational constraints of the expected -production environment. -Over the years the group has developed a diverse portfolio of -data system experience across a broad spectrum of domains -including earth and climate science (Mattmann et al. 2009; -Hart et al. 2011; Tran et al. 2011), planetary science, -astrophysics, snow hydrology, radio astronomy, cancer -research (Crichton et al. 2001), and pediatric intensive care -(Crichton et al. 2011). -2.3. Open Source and OODT -One of the products of this long track record of experience in -the realm of scientific data processing systems is a suite of -software components known as OODT1 originally arose out of -a desire on the part of NASAʼs Office of Space Science to -improve the return on investment for individual mission data -systems by leveraging commonalities in their design to create a -reusable platform of configurable components, on top of which -mission-specific customizations could be made. OODT thus -represents both an architecture and a reference implementation. -Its components communicate with one another over standard, -open protocols such as XML-RPC2 and can be used either -individually, or coupled together to form more complex data -processing pipelines. -In 2009 OODT began the transition from a JPL-internal -development project to a free and open source software project -at the Apache Software Foundation (ASF).3 Graduating to a -top-level project in 2011, OODT has since undergone several -public releases at the ASF and is in use by a varied group of -scientific and commercial endeavors. As we will describe -further in Section 3, several OODT components form the core -platform of our candidate validation framework. The ready -availability of OODT components under a liberal license, -combined with their substantial pedigree was appealing to our -project both for time and budgetary considerations. -2.4. Related Work -In the following section we identify several ongoing efforts -that also utilize online tools to assist in the collaborative review -and classification of scientific observations. -2.4.1. Astropulse -Astropulse is part of a series of sky surveys for radio pulses -being conducted by the Search for Extraterrestrial Intelligence -(SETI) at the University of Berkeley (Siemion et al. 2010). -The Astropulse project conducts a survey of the sky from the -Arecibo Observatory in Puerto Rico, searching for short -(microsecond) broadband radio frequency pulses. While -Astropulseʼs use of Areciboʼs enormous single dish telescope -affords excellent sensitivity, V-FASTRʼs ability to perform -continent-scale baseline interferometery yields much greater -positional accuracy when attempting to localize the source of a -signal. -As a variant of the SETI@home project, Astropulse utilizes the -same distributed, collaborative volunteer computing infrastructure -accumulated over the years by that effort to perform a -number of computationally intense transformations and calculations -of the data in an attempt to better classify the origin of any -signals detected. The use of volunteer computing to perform units -of computational work is an appealing approach that obviates the -need to directly acquire sufficient hardware for the processing -demands. However, the fully automated nature of the approach is -not a natural fit for V-FASTRʼs manual review requirement. -2.4.2. Galaxy Zoo -GalaxyZoo4 is an Internet-based project that relies on the -help of volunteers to classify a very large database of galaxy -images recorded by either the Sloan Digital Sky Survey or the -Hubble telescope. Users are asked to classify galaxies based on -1 Apache OODT: http://oodt.apache.org/ -2 XML-RPC: http://xmlrpc.scripting.com/spec.html -3 http://apache.org/ -4 Galaxy Zoo: http://www.galaxyzoo.org/ -2 -The Astronomical Journal, 149:23 (7pp), 2015 January Hart et al. -shape, color and direction of rotation, and report on possible -unidentified features. The rationale behind human intervention -is that manual classification is more accurate and insightful -than any algorithm that can currently by undertaken by an -automatic program. To date, the project has met with success -that far exceeded expectations: more than 250,000 volunteers -have helped classify millions of images, resulting in the -confirmation of important scientific hypothesis, the formulation -of new ones, and the discovery of new interesting objects. -While Galaxy Zooʼs tactic of appealing to volunteers to -mitigate the challenge of image classification at scale is -attractive, the paradigm does not translate well to the V-FASTR -setting due to differences in the nature of the archives between -the two projects. Whereas Galaxy Zoo permits its volunteer -reviewers to leisurely peruse and mine a largely static image -archive, the rapidly growing data volumes associated with -ongoing V-FASTR observations dictate that reviews must be -regularly scheduled to keep the project within its resource -limits. -2.4.3. Foldit: The Protein Folding Game -Foldit (Cooper et al. 2010) is a collaborative online protein -folding game developed by the Center for Game Science -at the University of Washington, and it represents a -“crowd-sourced” attempt to solve the computationally challenging -task of predicting protein structure. Proteins, chains of -amino acids, play a key role in a wide range of human diseases, -but comparatively little is known about how they contort -themselves into the specific shapes that determine their -function. Because of the scale and complexity of the challenge, -the researchers behind Foldit have turned to the puzzle-solving -capabilities of human beings for assistance. After learning the -rules on simple challenges, players compete against one -another to design alternative protein structures, with the goal -of arriving at an arrangement that minimizes the total energy -needed to maintain the shape. -Foldit has created an environment in which the unknown and -diverse strategies of its human participants become a core -strength. Furthermore, by presenting the scientific activity as a -competitive game, the project, which currently boasts over -400,000 players, has shown that it is possible to recruit and -leverage human processing power at scale. This provides an -interesting model for other projects, including V-FASTR, -which at some point may rely upon a human element to -augment or improve automated processes. -3. IMPLEMENTATION -In this section we provide details on the implementation of -our metadata-driven framework for online review of V-FASTR -candidate detection events. We describe our methodology and -the considerations that informed our design, followed by a -presentation of the system architecture. -3.1. Development Methodology -Several factors influenced the development process and have -left their imprint on the final architecture. We feel that our -implementation is uniquely suited to the needs of the VFASTR -project precisely because these factors were identified -early on and were thus able to influence all aspects of the -design process. -3.1.1. Collaboration -As described in Section 2, our group has developed -substantial experience in the design and implementation of -data systems for a broad range of scientific domains. In each -case, a close working relationship with members of the project -science team was an essential ingredient to the success of the -project, and our experience developing an online candidate -review framework for V-FASTR was no different. As software -engineers familiar with the challenges inherent in scientific data -management, our intuitions about the technical challenges of -the system served us well in scoping out the project timeline. -However, it was our early and regular communication with -members of the V-FASTR science team that was critical to -obtaining the domain knowledge necessary to make accurate -assumptions, and in the early identification of issues. The -current system architecture, covering both the back and front -end elements, is a direct result of an ongoing feedback loop -between the science and software teams. -3.1.2. Constraints -As mentioned in Section 2, V-FASTR is a commensal -experiment that scans for fast transients in data that is already -being collected as part of the regular third-party use of the -VLBA instrument. As such, the experiment maintains a “guest” -status on the NRAO computing infrastructure. Consequently, -care must consistently be taken not to overtax NRAO system -resources, including disk storage, CPU time, and network -bandwidth. These physical constraints motivated many of the -architectural decisions described in the following sections. -Each V-FASTR data product may contain hundreds of files, -rooted at a top-level job directory, and includes two types of -products: filterbank data (up to ~100 GB per job) and -baseband voltage data (up to ~10 GB per job). The total data -storage capacity available to V-FASTR is just ~8 TB, enough -to contain ~800 jobs of ~10 GB each (on average). Because -products are produced at a average rate of ~10–20 per day (but -sometimes in the hundreds), the storage would be exhausted -within a few weeks unless products are periodically reviewed -by the science team analysts. During review, each candidate is -either flagged for higher-resolution processing (and saved) or -discarded as a false positive and the disk space reclaimed (see -Figure 1 for an overview of the average data volumes per job at -different processing stages). The desire to provide analysts with -a streamlined method for this review process is at the very core -of our design. -Similarly, the network bandwidth constraints of the host led -us to a data transfer configuration that focused on metadata -rather than requiring the complete transfer of raw, unreviewed, -and possibly spurious detection data over the Internet. Instead, -metadata sufficient to describe the salient characteristics of a -candidate event to a trained analyst was transferred into our -candidate review framework. This careful selection process had -the beneficial side effect of greatly limiting the size of the -transferred products, allowing for a considerably longer -retention period on the ~10 TB archive hosted at JPL. -Finally, security constraints were also critically important to -the design, particularly because the system spans two separate -security domains: NRAO and JPL. To comply with the security -requirements of the host system, data transfer was configured -on the NRAO system to allow read-only operations and was -made accessible only to clients originating from the JPL -3 -The Astronomical Journal, 149:23 (7pp), 2015 January Hart et al. -domain. Furthermore, on the front-end, the functionality -exposed by the web portal component interacted only with -the local metadata archive, eliminating the possibility of -corruption or inappropriate access to the raw observational -data. -3.2. Architecture -As previously mentioned, the candidate review framework is -driven by metadata describing the candidate events to be -reviewed by V-FASTR analysts. To communicate this data -from the raw source repository at the NRAO to an analyst using -a browser anywhere in the world, we developed a software -framework consisting of two principal components: a metadata -pipeline that manages the capture, transfer, and storage of -metadata annotations, and a web portal which provides analysts -with a convenient, context-rich environment for efficiently -classifying candidate events. -3.2.1. Metadata Pipeline -On the JPL side, the V-FASTR data products are processed -through a metadata extraction and data archiving pipeline that -eventually leads to the event candidates being available for -inspection on the web portal. The pipeline is composed of three -major software components: rsync, the OODT CAS Crawler, -and the OODT File Manager, depicted in Figure 2. -rsync. Data products are automatically transferred from the -NRAO staging area to the JPL server using rsync. rsync is a -popular application and data transfer protocol that allows to -synchronize the content of a directory tree between two -servers with minimal human intervention. It was chosen -because of its simplicity, high performance, reliability, and -wide range of configuration options. Through rsync, files are -transferred in compressed format and using delta encoding, -meaning that only the file differences are sent through -subsequent transfers. For this project, an rsync server -daemon was set up on the NRAO side to expose the data -staging area where the products are collected. For security -reasons, the daemon was restricted to allow read-only -operations to clients originating from a designated JPL IP -address. On the JPL side, an rsync client was set up to run -hourly as a system cron job, transferring products to the JPL -archive area. To minimize bandwidth usage, the client only -transfers a very small subset of the data comprising a product -directory tree, namely the detection images and the output -and calibration files containing the metadata needed by the -web portal. On average, this represents a reduction of the -data product size by a factor of 3.5 ´ 103: from an average -size of ~35 GB on the NRAO server (for a product with -several detections), to ~10 MB on the JPL server. The rsync -data transfer rates between the two servers were measured to -be around ~2 MBs-1, more than enough to transfer between -10 and 20 data products per day. -CAS Crawler. Once the data products are transferred to the -JPL server, they are automatically detected by the OODT -CAS Crawler daemon, which runs at sub-hour time intervals -to pick up new products as soon as they become available. -The Crawler is responsible for notifying the OODT File -Manager and therefore starting the product ingestion process. -For this deployment, the Crawler was configured to send a -signal only if two preconditions are both satisfied: (1) a -similarly named product does not already exist in the File -Manager catalog and (2) the product directory contains a -special marker file indicating that the product has been -processed by the mail program, and therefore is in a complete -state (i.e., no files are missing). -CAS File Manager. The OODT CAS File Manager is a -customizable software component that is responsible for -processing and archiving a data product, making it available -for query and access to clients. For this project, the File -Manager was deployed with the default Apache Lucene -metadata back-end, and configured to archive products -Figure 1. Depiction of the full V-FASTR data flow with volume estimates (per job) at each stage. The candidate review framework (both metadata pipeline and web -portal components) interact with the metadata and derived products repository at the intersection of A and B above. -4 -The Astronomical Journal, 149:23 (7pp), 2015 January Hart et al. -in-place, i.e., without moving them to a separate archive -directory, otherwise the rsync process would transfer them -again from the NRAO server. Additionally, we leveraged the -extensibility of the OODT framework by configuring the File -Manager with custom metadata extractors that were -purposely written to parse the information contained in the -V-FASTR output and calibration files. Information is -extracted at the three levels that comprise the hierarchy of -a V-FASTR data product: job, scan, and event. Additionally, -a numerical algorithm was written to assign each pair of -available images (-det.jpg and -dedisp.jpg) to the event that -generated them. -In general, a File Manager can store metadata in its back-end -catalog as different object types. Each object type is defined to -contain multiple metadata fields, where each field is composed -of a named key associated to one or more string values. For this -project, the decision was made to maintain a one-to-one -correspondence between a data product and the corresponding -metadata ingested into the catalog. So rather than defining three -object types for jobs, scans, and events, a single object type -was used holding all information for a data product in a single -container, with dynamically named keys that are encoded to -contain the scan and event numbers. This decision was -motivated by the desire to simplify and optimize the querying -of information by the web portal client, since all metadata for a -product is retrieved through a single request to the File -Manager. As a consequence, the default Apache Lucene -metadata catalog implementation had to be slightly modified -to allow for the ingestion of dynamically named metadata -fields. -3.2.2. Web Portal -The second major component of the candidate review -framework is an interactive web portal. The primary purpose -of the portal is to provide a convenient online environment for -the location-independent perusal and assessment of potential -candidates in context. The portal provides V-FASTR analysts -with the ability to quickly navigate through the available -information to identify candidates worthy of further inspection -on a familiar web platform. -The portal has been implemented as a PHP web application -using the Apache OODT Balance web framework running on -top of the Apache HTTPD Web Server. OODT Balance was -chosen here for its ability to easily integrate with the OODT -components in the back-end metadata pipeline, namely the -OODT CAS File Manager described earlier. Furthermore, the -flexible, modular approach of the framework allowed us to -quickly connect the web portal to the metadata repository and -rapidly begin constructing the necessary views specific to the -V-FASTR candidate review and validation use cases. -As Figure 3 shows, the web portal offers a variety of views -of the available metadata which are hierarchically organized to -match the conceptual relationships in the data. At the highest -level, a job or run might consist of multiple scans, each of -which may itself contain multiple detection event candidates. -This hierarchy, expressed in the metadata, is preserved in the -layout of the portal views, and the breadcrumb navigation -provided to facilitate orientation within the nested structure. -At the level of an individual event candidate (Figure 3, -middle image), two graphical representations of the event are -available to assist analysts in classifying the nature of the -signal. These images are generated automatically as part of the -initial candidate identification process (Wayth et al. 2011), and -they provide a trained analyst the necessary structural clues -needed to rapidly assess the received signal as being genuinely -extraterrestrial in origin or merely a product of RFI. -To support both metadata browsing in context and the desire -for an analyst to be able to rapidly peruse the image -representations of an entire job (many events in many scans) -at once, a compromise was struck whereby, for each job, a -portal user may select a traditional, hierarchical navigation or a -flattened view in which all of the (possibly numerous) event -candidates are presented simultaneously on screen and can be -accessed serially simply by scrolling the view. -Figure 2. Component diagram for the metadata pipeline component of the VFASTR -candidate review framework. -5 -The Astronomical Journal, 149:23 (7pp), 2015 January Hart et al. -Together, the metadata pipeline and the web portal constitute -an end-to-end framework for capturing, archiving, and -presenting metadata about detected transient event candidates -to V-FASTR scientists. Furthermore, by providing a reliable -process and flexible interface, the system directly streamlines -the analysis process, boosting the overall efficiency of the -project. -4. EVALUATION -As we have described in the previous section, the candidate -review framework embraces the model of online collaborative -validation of fast transient candidates by a team of geographically -dispersed analysts, and improves the efficiency with -which analysts may classify observational data. In this section -we describe the early results of our experience with the -operational deployment of the framework, as well as highlight -several areas for the evolution of the tool to further enhance its -utility. -4.1. Experience -The initial deployment of the collaborative review framework -for operational use by the V-FASTR science team was -made in early summer 2012. The immediate feedback was -largely positive: analysts praised the capabilities of the system, -the general improved accessibility afforded by a web-based -user interface, and the newfound capability to easily navigate -rapidly through all detections in a given job, or peruse the -different levels (scans and events) within a job individually. -The biggest initial complaint with the system was that too -many mouse clicks were required to complete an analysis of all -of the candidates in an entire job. -A consequence of the iterative feedback loop that developed -between the software and science teams (described further in -Section 3) was that suggestions for improvements were -repeatedly made, tracked, and acted upon. This process resulted -in an updated release occurring approximately every two weeks -during the first few months of the deployment. Suggestions for -improvements included the addition of various metadata fields -identified as critical to the classification task, updates to the -visual organization of the elements of the web portal views, and -a relentless focus on reducing the number of mouse clicks -required on the part of analyst users. -By the time of this writing, the V-FASTR portal has been -running operationally for several weeks, and we can draw some -early conclusions on usefulness of the system. Overall, as -reported by the science team, it seems like the project has -definitely accomplished its broad goal of facilitating the -collaborative task of inspecting and screening radio-transient -events. By extracting all relevant metadata from the latest data -products, and presenting it on the web portal in a concise -fashion, scientists can now execute their tasks more efficiently, -compared to earlier times when they had to log onto a terminal -and analyze the raw data manually. Additionally, the online -availability of all data and metadata through a browser interface -(as opposed to an ssh terminal) has allowed for greater -flexibility with regard to when and where evaluations can be -performed, including for the first time on a mobile device. -4.2. Evolution -On the whole, the ability to interact with the totality of the -candidate data and metadata through a browser interface has -greatly expanded the analysts’ ability to perform their tasks -with greater flexibility regarding when and where evaluations -can be performed. This includes, for the first time, anecdotal -accounts of an analyst reviewing candidates from a mobile -device. -With this freedom, in turn, has come a number of feature -requests which can be taken together to form a roadmap of -sorts for the evolution of the framework. Now that the -interaction with candidate metadata has transitioned to the -browser, the science team has identified three key features they -feel would complete the transition and entirely replace the prior -ad-hoc methods for coordinating the analysts’ activities: -Job assignment. As mentioned in Section 3, the timely -review of detection candidates is critical to remaining within -the resource constraints imposed upon the experiment. At the -moment, review jobs are assigned to analysts via email. -Augmenting the web portal with the ability to identify an -individual analyst would enable the presentation of -a prioritized list of that analystʼs outstanding review tasks. -Effort Tracking. Along the same lines, it is important to -spread the analysis load evenly across the science team, since -no one person is performing the analysis as his or her fulltime -job. Augmenting the framework with the ability to track -the analysis contributions of individual users over time -would assist in the equitable scheduling of future -review jobs. -Figure 3. Screen shots of the initial version of the web portal component. From left to right: the portal home page displaying recent jobs and associated event counts, -image metadata associated with an individual event candidate, full metadata listing, including associated scans, for an observation job. -6 -The Astronomical Journal, 149:23 (7pp), 2015 January Hart et al. -In-browser archiving. When an analyst determines a -candidate event merits high-resolution followup, the last -step is to archive the associated raw data so that it can be -evaluated at a later date. Currently, due to the security -restrictions permitting read-only access to external connections -to the archive at the NRAO (described in Section 3), -this process is handled out-of-band by the analyst logging -into an NRAO machine and archiving the appropriate data -manually. It is possible that, with the identity management -features discussed in the previous two items (and the -associated auditing capabilities that it could entail) the -restrictions might be negotiated to the point that certain -defined activities (such as archiving a single job directory) -could be initiated from within the portal environment. -5. CONCLUSION -V-FASTR, and commensal operations more generally, are -particularly challenging experiments due to extreme data -volume and real-time requirements. Processing occurs continually, -and the data flow must be coordinated across multiple -physical locations with transport mechanisms ranging from -FedEx transport (disks from the antenna), high-bandwidth -interconnects (the correlator and transient detection systems), -daily rsync over IP (the ska-dc mirror), and distributed WWW -protocols (manual review which takes place by analysts on -three continents). Various components of the system operate on -millisecond, hourly, and daily clocks and all components must -continue operating since there is very little margin for buffer -resources. In addition, the data processing components are -highly heterogeneous, with human experts playing their own -role as scheduled pattern recognition engines in the overall -architecture. By facilitating timely review, and reducing the -learning curve for new reviewers, the V-FASTR portal will -play a critical role in keeping the data flowing and making the -system sustainable in the long term. -This effort was supported by the Jet Propulsion Laboratory, -managed by the California Institute of Technology under a -contract with the National Aeronautics and Space -Administration. -REFERENCES -Cooper, S., Khatlib, F., & Treuille, A. 2010, Natur, 466, 756–60 -Cordes, J., Lazio, T., & McLaughlin, M. 2004, NewAR, 48, 1459–72 -Crichton, D., Kincaid, H., Downing, G., Srivastava, S., & Hughes, J. S. 2001, -in Proc. of the 14th IEEE Symp. on Computer-Based Medical Systems, An -Interoperable Data Architecture for Data Exchange in a Biomedical -Research Network (Piscataway, NJ: IEEE), 65–72 -Crichton, D., Mattmann, C., Hart, A., et al. 2011, in Proc. of the 24th -IEEE Symp. on Computer-Based Medical Systems An Informatics -Architecture for the Virtual Pediatric Intensive Care Unit (Piscataway, -NJ: IEEE), 1–6 -Hart, A., Goodale, C., Mattmann, C., et al. 2011, in Proc. of the 2nd Int. -Workshop on Software Engineering for Cloud Computing, A Cloudenabled -Regional Climate Model Evaluation System (New York: -ACM), 43–49 -Mattmann, C., Crichton, D., Medvidivic, N., & Hughes, J. S. 2006, in Proc. -2006 Int. Conf. on Software Engineering, A Software Architecture-based -Framework for Highly Distributed and Data Intensive Scientific -Applications (New York: ACM), 721–30 -Mattmann, C., Freeborn, D., Crichton, D., et al. 2009, in Proc. IEEE Int. Conf. -on Space Mission Challenges for Information Technology, A Reusable -Process Control System Framework for the Orbiting Carbon Observatory -and NPP Sounder PEATE Missions (Piscataway, NJ: IEEE), 165–72 -Romney, J. D. 2010, NRAO, http://www.vlba.nrao.edu/astro/obstatus/current/ -obssum.html. -Siemion, A., von Korff, J., McMahon, P., Korpela, E., & Werthimer, D. 2010, -AcAau, 67, 1342–9 -Thompson, D., Wagstaff, K., Brisken, W., et al. 2011, ApJ, 735, 98 -Tran, J., Cinquini, L., Mattmann, C., et al. 2011, in Evaluating Cloud -Computing in the NASA DESDynI Ground Data System Proc. of the II -International Workshop on Software Engineering for Cloud Computing -(New York: ACM), 36–42 -Wayth, R., Brisken, W., Deller, A., et al. 2011, ApJ, 735, 97 -Wayth, R., Tingay, S., & Deller, A. 2012, ApJL, 753, L36 -7 -The Astronomical Journal, 149:23 (7pp), 2015 January Hart et al. \ No newline at end of file diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/A FRAMEWORK FOR COLLABORATIVE REVIEW OF CANDIDATE EVENTS IN HIGH DATA RATE STREAMS THE V-FASTR EXPERIMENT AS A CASE STUDY.txt.xml.xls b/src/main/resources/sdtocode/doc/Apache OODT File Manager/A FRAMEWORK FOR COLLABORATIVE REVIEW OF CANDIDATE EVENTS IN HIGH DATA RATE STREAMS THE V-FASTR EXPERIMENT AS A CASE STUDY.txt.xml.xls deleted file mode 100644 index 00500ea9ef89f7c35d8d152086a7cd8b74a0c1a7..0000000000000000000000000000000000000000 Binary files a/src/main/resources/sdtocode/doc/Apache OODT File Manager/A FRAMEWORK FOR COLLABORATIVE REVIEW OF CANDIDATE EVENTS IN HIGH DATA RATE STREAMS THE V-FASTR EXPERIMENT AS A CASE STUDY.txt.xml.xls and /dev/null differ diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/A Reusable Process Control System Framework for the Orbiting Carbon Observatory and NPP Sounder PEATE missions-relation.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/A Reusable Process Control System Framework for the Orbiting Carbon Observatory and NPP Sounder PEATE missions-relation.txt deleted file mode 100644 index 9871327f3ddbe39738ab1f1a713e4551a4e6f596..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/doc/Apache OODT File Manager/A Reusable Process Control System Framework for the Orbiting Carbon Observatory and NPP Sounder PEATE missions-relation.txt +++ /dev/null @@ -1,591 +0,0 @@ -page&Paul Ramirez&依赖 -author&publication&AGGREGATION -page&22 may 2014&依赖 -enhancement&downloaded file&AGGREGATION -user&downloaded file&依赖 -user&enhancement&依赖 -Charles E. Miller Jet Propulsion Laboratory California Institute&Technology Pasadena&AGGREGATION -We&science processing pipeline&实现 -We&reusable architecture and implementation framework&依赖 -We&mission ground data system&实现 -system&software component&依赖 -Our&system& -system&software component&依赖 -archive ( cas )&QuikSCAT , SeaWinds and AMT earth science mission&依赖 -software component&component&GENERALIZATION -paper&orbiting carbon observatory (&依赖 -paper&pc&依赖 -paper&two current earth science mission&依赖 -context&two current earth science mission&AGGREGATION -paper&context&依赖 -order&magnitude&AGGREGATION -1 Introduction Data volume&order&依赖 -1 Introduction Data volume&magnitude&依赖 -1 Introduction Data volume&magnitude&依赖 -1 Introduction Data volume&magnitude&依赖 -1 Introduction Data volume&order&依赖 -1 Introduction Data volume&order&依赖 -increase power and pervasiveness&high performance computing&AGGREGATION -low cost&era&依赖 -low cost&disk storage space&AGGREGATION -low cost&era&依赖 -previously unimaginable science question&era&依赖 -previously unimaginable science question&year&依赖 -science question&sea surface temperature&依赖 -Earth&cycle& -better understanding&’s global carbon cycle and climate change&AGGREGATION -study&sea surface temperature&AGGREGATION -science question&study&依赖 -significant portion&space-based NASA earth science mission&AGGREGATION -goal&1A strategically&AGGREGATION -several focused series&step&AGGREGATION -processing&mundane activity&依赖 -processing&datum&依赖 -prospective and retrospective physical modeling&scene&AGGREGATION -processing&marshalling&依赖 -processing&datum&依赖 -processing&mundane activity&依赖 -processing&marshalling&依赖 -( removal&special space “ header ” information )&AGGREGATION -our&work& -central data base ( cdb ) subsystem&datum base management software&依赖 -datum base management software¤t and future planetary mission&依赖 -central data base ( cdb ) subsystem&mgd&AGGREGATION -central data base ( cdb ) subsystem&datum base management software&依赖 -datum base management software¤t and future planetary mission&依赖 -central data base ( cdb ) subsystem&datum base management software&依赖 -central data base ( cdb ) subsystem&datum base management software&依赖 -architectural principle&extensibility&AGGREGATION -architectural principle&ammo&依赖 -part&larger system&AGGREGATION -architectural principle&CDB&AGGREGATION -CDB&controlled , centralized system&依赖 -part&controlled , centralized system&AGGREGATION -work&clear two significant trend&依赖 -Our&work& -work&clear two significant trend&依赖 -developer&method&依赖 -developer&method&依赖 -we&“ rapid adaptation ”&依赖 -we&original CDB software&依赖 -“ rapid adaptation ”&original CDB software&AGGREGATION -technology task&task&GENERALIZATION -technology task&time&依赖 -NASA Office&Space Science&AGGREGATION -“ out&box ” core data management software service&AGGREGATION -Several author&this paper support seawind&AGGREGATION -result&collaboration&AGGREGATION -CAS component&component&GENERALIZATION -their&systems& -QuikSCAT&planned 2-year mission&依赖 -its&year& -10th year&planned 2-year mission&AGGREGATION -reprocessing and simultaneous&7 year&AGGREGATION -7 year&datum&AGGREGATION -JPL earth mission&increased data volume&依赖 -CAS component&Sea&依赖 -JPL earth mission&increased data volume&依赖 -JPL earth mission&far more complex processing (&依赖 -CAS component&Winds and AMT&依赖 -JPL earth mission&far more complex processing (&依赖 -development&a third component&AGGREGATION -refactoring&CAS component&AGGREGATION -refactoring&several issue&依赖 -refactoring&several issue&依赖 -refactoring&several issue&依赖 -refactoring&several issue&依赖 -refactoring&CAS&AGGREGATION -it&ingestion&依赖 -ingestion&file&AGGREGATION -initiation&workflow&AGGREGATION -it&initiation&依赖 -it&file&依赖 -it&workflow&依赖 -specific time&day&AGGREGATION -ingestion&particular file or file type&AGGREGATION -independent evolution&component&AGGREGATION -refactoring&component&依赖 -refactoring&independent evolution&依赖 -combination&three refactored CAS component&AGGREGATION -we&File Manager , Workflow Manager and Resource Manager component&依赖 -we&component&依赖 -we&reusable interface&依赖 -we&reusable interface&依赖 -we&configurable push-pull framework and crawler framework&依赖 -phase&algorithm development&AGGREGATION -easy integration&science code&AGGREGATION -they&future&依赖 -they&Earth science mission work&依赖 -we&core PCS component&依赖 -their&architecture& -future&Earth science mission work&AGGREGATION -we&architecture&依赖 -we&architecture&依赖 -they&us&依赖 -they&OCO&依赖 -our&components& -they&problem&依赖 -we&core PCS component&依赖 -its&evolution& -We&same spirt&依赖 -We&architectural reuse , understanding and mission specific adaptation&依赖 -genesis&modern pc&AGGREGATION -same spirt&architectural reuse , understanding and mission specific adaptation&AGGREGATION -our&uniquely& -state&art&AGGREGATION -We&us&依赖 -rest&paper&AGGREGATION -rest&follow&依赖 -area&grid computing , workflow systems and science data system&AGGREGATION -Section 3&greater detail&依赖 -Section 3&core PCS architectural component&依赖 -Section 4&our experience&依赖 -Section 4&leveraging&依赖 -our&experience& -section 5 round&paper&依赖 -section 5 round&paper&依赖 -development&virtualization and sharing&依赖 -development&organizational and geographic boundary&依赖 -virtualization and sharing&processing and storage resource&AGGREGATION -development&8 ]&依赖 -development&8 ]&依赖 -many groups and organization&power&依赖 -development&virtualization and sharing&依赖 -development&virtualization and sharing&依赖 -development&8 ]&依赖 -development&organizational and geographic boundary&依赖 -many groups and organization&grid&依赖 -development&8 ]&依赖 -development&8 ]&依赖 -development&8 ]&依赖 -development&processing and storage resource&依赖 -development&processing and storage resource&依赖 -development&processing and storage resource&依赖 -development&8 ]&依赖 -development&virtualization and sharing&依赖 -development&organizational and geographic boundary&依赖 -power&grid&AGGREGATION -development&8 ]&依赖 -many groups and organization&power&依赖 -development&processing and storage resource&依赖 -enabler&large-scale scientific research&AGGREGATION -many groups and organization&power&依赖 -development&processing and storage resource&依赖 -development&organizational and geographic boundary&依赖 -development&processing and storage resource&依赖 -development&processing and storage resource&依赖 -development&virtualization and sharing&依赖 -development&8 ]&依赖 -development&8 ]&依赖 -development&8 ]&依赖 -development&8 ]&依赖 -development&organizational and geographic boundary&依赖 -development&organizational and geographic boundary&依赖 -development&virtualization and sharing&依赖 -development&8 ]&依赖 -many groups and organization&grid&依赖 -development&organizational and geographic boundary&依赖 -development&8 ]&依赖 -development&computational grid [&AGGREGATION -development&virtualization and sharing&依赖 -many groups and organization&grid&依赖 -we&ongoing software project&依赖 -we&paper&依赖 -collection&open-source software tool&AGGREGATION -2.1 Grid Systems The Globus toolkit [ 9 ]&open-source software tool&依赖 -2.1 Grid Systems The Globus toolkit [ 9 ]&open-source software tool&依赖 -many aspect&a distribute , service-oriented infrastructure include security , resource and datum discovery&AGGREGATION -communication module&particular gridbased effort&依赖 -suite&software components and library&AGGREGATION -2.2 Workflow Systems&number&依赖 -2.2 Workflow Systems&number&依赖 -2.2 Workflow Systems&explosion&依赖 -2.2 Workflow Systems&workflow languages and software system&依赖 -2.2 Workflow Systems&workflow languages and software system&依赖 -2.2 Workflow Systems&explosion&依赖 -number&workflow languages and software system&AGGREGATION -Yu and Buyya [ 15 ]&scientific workflow system&依赖 -they&technology&依赖 -Woollard and et . al.&workflow system&依赖 -Woollard and et . al.&characterization&依赖 -Woollard and et . al.&characterization&依赖 -Woollard and et . al.&workflow system&依赖 -Woollard and et . al.&characterization&依赖 -Woollard and et . al.&workflow system&依赖 -Woollard and et . al.&workflow system&依赖 -Woollard and et . al.&workflow system&依赖 -Woollard and et . al.&workflow system&依赖 -Woollard and et . al.&characterization&依赖 -Woollard and et . al.&characterization&依赖 -Woollard and et . al.&characterization&依赖 -Woollard and et . al.&workflow system&依赖 -characterization&workflow system&AGGREGATION -Woollard and et . al.&workflow system&依赖 -Woollard and et . al.&characterization&依赖 -Woollard and et . al.&characterization&依赖 -author&Production Systems&依赖 -OCO and NPP Sounder PEATE ground data system&which&依赖 -author&certain workflow system&依赖 -example&which&AGGREGATION -effective usage&available distributed computing and storage resource&AGGREGATION -University&Wisconsin Madison&AGGREGATION -Condor&user&依赖 -their&execution& -Condor&user&依赖 -Condor&infrastructure&依赖 -notification&completion&AGGREGATION -Condor&detail&依赖 -Condor&transparently&依赖 -detail&infrastructure&AGGREGATION -particulars&detail&AGGREGATION -variant&Condor and Condor-G [ 10 ]&AGGREGATION -variant&Globus toolkit&依赖 -variant&Globus toolkit&依赖 -variant&Globus toolkit&依赖 -layer&abstraction&AGGREGATION -large number&grid-based compute node&AGGREGATION -Pegasus&workflow&依赖 -Pegasus&workflow&依赖 -event&failure&AGGREGATION -small number&compute node&AGGREGATION -Pegasus&ability&依赖 -Pegasus&ability&依赖 -Pegasus&ability&依赖 -Pegasus&workflow&依赖 -reliability&system&AGGREGATION -vast quantity&intermediate and final data product&AGGREGATION -base level&service&AGGREGATION -large number&system&AGGREGATION -our&discussion& -support&operation&AGGREGATION -operation&Goddard Earth Science Data&AGGREGATION -developer&pain&依赖 -cost&development&依赖 -cost&S4PA&依赖 -development&S4PA&AGGREGATION -primary copy&all datum online&AGGREGATION -its&name& -favor&first-class architectural component and connector&AGGREGATION -large number&complimentary , interconnected service&AGGREGATION -complete science data processing pipeline&complimentary , interconnected service&依赖 -complete science data processing pipeline&large number&依赖 -We&Open Grid Framework&依赖 -2.4.1 OGF The Open Grid Forum [ 2 ]&standards and specification&依赖 -2.4.1 OGF The Open Grid Forum [ 2 ]&grid-based software system&依赖 -2.4.1 OGF The Open Grid Forum [ 2 ]&standards and specification&依赖 -2.4.1 OGF The Open Grid Forum [ 2 ]&adoption&依赖 -adoption&grid-based software system&AGGREGATION -OGF&business , government , scientific , and academic organization&依赖 -utilization&grid&AGGREGATION -OGF&role&依赖 -OGF&grid&依赖 -role&grid&AGGREGATION -OGF&independent voice&依赖 -OGF&grid&依赖 -their&potential& -OGF&independent voice&依赖 -OGF&role&依赖 -2.4.2 OAI The Open Archives Initiative [ 1 ]&interoperability&依赖 -2.4.2 OAI The Open Archives Initiative [ 1 ]&standard&依赖 -goal&application interoperability&依赖 -goal&application interoperability&依赖 -application interoperability&interoperability&GENERALIZATION -goal&OMI-PMH&AGGREGATION -OAI&broader accessibility and usability&依赖 -broader accessibility and usability&distributed data resource&AGGREGATION -rich array&application-specific metada&AGGREGATION -OAI&distributed data resource&依赖 -we§ion&依赖 -we&PCS core component&依赖 -we&3 PCS Core Architecture&依赖 -2 ) managing pipeline processing&PCS Task Wrapper&依赖 -2 ) managing pipeline processing&PCS Task Wrapper&依赖 -three PCS framework&two critical higher level service&实现 -one&two critical higher level service&AGGREGATION -three PCS framework&one&实现 -three PCS framework&two critical higher level service ###&实现 -We&component&依赖 -We&below&依赖 -We&greater detail&依赖 -overall PCS architecture&Fig. 1&依赖 -critical object&Products&依赖 -critical object&collection&依赖 -collection&one or more file&AGGREGATION -critical object&one or more file&依赖 -critical object&one or more file&依赖 -critical object&Products&依赖 -critical object&collection&依赖 -their&Metadata& -A map&key&AGGREGATION -multiple value&descriptive information&AGGREGATION -its&location& -file&location& -File Manager&use&依赖 -File Manager&what type&依赖 -what type&file uniform resource identifier ( uri ) [ 5 ] generation scheme&AGGREGATION -File Manager&file uniform resource identifier ( uri ) [ 5 ] generation scheme&依赖 -description&Product&AGGREGATION -File Manager&use&依赖 -File Manager&file uniform resource identifier ( uri ) [ 5 ] generation scheme&依赖 -element&form&依赖 -element&associated definition&依赖 -form&associated definition&AGGREGATION -element&additional metada&依赖 -A URI generation scheme&location&依赖 -A URI generation scheme&built&依赖 -A URI generation scheme&archive (&依赖 -Product&one or more reference&依赖 -member&single Product Type&AGGREGATION -Product&single Product Type&依赖 -Metadata&mapping&依赖 -mapping&Product Type&AGGREGATION -Metadata&one or more element&依赖 -Product Type&associated Versioner&依赖 -description , execution , and monitoring&Workflows&AGGREGATION -3.2 Workflow Manager TheWorkflow Manager component&Workflows&依赖 -3.2 Workflow Manager TheWorkflow Manager component&description , execution , and monitoring&依赖 -sequence&task&AGGREGATION -Workflows&output datum&依赖 -business&practices& -event&what triggerworkflow&依赖 -dynamic set&property&AGGREGATION -description&control flow&AGGREGATION -data flow&sequence&AGGREGATION -instance&Workflow&AGGREGATION -WorkflowInstance&shared Metadata context&依赖 -who&theWorkflow&依赖 -description&data flow&AGGREGATION -part&Workflow&AGGREGATION -static configuration property&WorkflowTask&依赖 -execution&WorkflowTask&AGGREGATION -Event&one or more Workflow Instances&依赖 -Workflow Instance&Workflow&依赖 -run-time execution model&Workflow&AGGREGATION -Workflow&one or more Workflow Tasks&依赖 -Workflow Task&single Workflow Task Configuration&依赖 -Workflow Task&model&依赖 -Workflow Task&a corresponding workflow task instance (&依赖 -Workflow Condition&corresponding Workflow Condition Instance&依赖 -3.3 Resource Manager The Resource Manager component&hardware resource&依赖 -set&hardware resource&AGGREGATION -3.3 Resource Manager The Resource Manager component&jobs , storage and networking resource&依赖 -3.3 Resource Manager The Resource Manager component&excecution , monitoring and traacking&依赖 -3.3 Resource Manager The Resource Manager component&set&依赖 -excecution , monitoring and traacking&jobs , storage and networking resource&AGGREGATION -abstract representation&execution unit&AGGREGATION -Job&that&依赖 -abstrct representation&input&AGGREGATION -its&Input& -complete specification&Job&AGGREGATION -physical code&job execution&依赖 -Job Spec&exactly one job&依赖 -Job&single Job Instance&依赖 -Job&one Resource Node&依赖 -satisfaction&ingestion pre-condition&AGGREGATION -type&files and directory&依赖 -type&files and directory&依赖 -type&activity&AGGREGATION -type&identification&依赖 -type&identification&依赖 -identification&files and directory&AGGREGATION -crawled datum&standard three state lifecycle&依赖 -crawled datum&standard three state lifecycle&依赖 -Crawler&customized precondition verification&依赖 -we&Product Crawlers&依赖 -Crawler&line&依赖 -three type&Product Crawlers&AGGREGATION -Crawler&crawilng strategy&依赖 -we&three type&依赖 -line&customized precondition verification&AGGREGATION -we&Product Crawlers&依赖 -we&three type&依赖 -file&process&依赖 -behavior&MetExtractorProductCrawler&AGGREGATION -AutoDetectCrawler&content type identification&依赖 -critical object&three phase&依赖 -ProductCrawler&given phase&依赖 -one or more&three phase&AGGREGATION -critical object&one or more&依赖 -Precondition Comparator -&MetExtractorProductCrawler and AutoDetectProductCrawler&依赖 -part&ProductCrawlers customized implementation&AGGREGATION -They&ProductCrawlers customized implementation&依赖 -They&precondition verification&依赖 -ProductCrawlers customized implementation&appropriate time&依赖 -ProductCrawlers customized implementation&precondition verification&AGGREGATION -protocol plugin&HTTP&实现 -implementation&remote protocol&AGGREGATION -one service&protocol layer&依赖 -one service&ProductCrawler&依赖 -3.5 push pull framework crawler framework&3.5 push pull framework crawler framework&依赖 -one service&further work&依赖 -protocol plugin&implementation&依赖 -development&protocol layer&AGGREGATION -protocol plugin&remote protocol&实现 -Push Pull Framework&modern web protocol&依赖 -Push Pull Framework&remote datum acquisition and acceptance&依赖 -Push Pull Framework&modern web protocol such ###&依赖 -email-based push datum acceptance&IMAP&依赖 -file&which&依赖 -Retrieval Method&manner&依赖 -file&manner&依赖 -file&remote site&依赖 -complexity&multi-threaded file download )&AGGREGATION -a fileretrievalsystem (&multi-threaded file download )&依赖 -configuration file&file&GENERALIZATION -a fileretrievalsystem (&complexity&依赖 -ListRetriever&given list&依赖 -ListRetriever&file URIs [ 5 ]&依赖 -given list&file URIs [ 5 ]&AGGREGATION -Virtual&filter uri&依赖 -Protocol -&transfer protocol&依赖 -Protocol -&file transfer and communication&依赖 -execution&NASA Product Generation executive&AGGREGATION -pge&step&依赖 -pge&overall scientific process&依赖 -pge&scientific algorithm&依赖 -its&lifecycle& -PGE&file , or metada&依赖 -manner&requirement&依赖 -its&requirements& -PGE&file&依赖 -sequence&execution&AGGREGATION -PGE&upstream or downstream pge&依赖 -PGE&knowledge&依赖 -knowledge&upstream or downstream pge&AGGREGATION -PGE&knowledge&依赖 -PGE&upstream or downstream pge&依赖 -PGE&upstream or downstream pge&依赖 -PGE&knowledge&依赖 -its&file& -output Product file and Metadata generation&PCS Task Wrapper framework&依赖 -PCS Task Wrapper&output Product file&依赖 -PCS Task Wrapper&Crawler Framework&依赖 -execution&step&AGGREGATION -generalized set&action&AGGREGATION -abstract class&generalized set&依赖 -abstract class&action&依赖 -Pge Config File Builder&PgeConfig object&依赖 -additional Metadata&information&依赖 -lifecycle&xml file&实现 -PCS Task Wrapper&simple but powerful XML syntax&依赖 -xml file&file&GENERALIZATION -lifecycle&scientist&实现 -Config File Property Adder&Pge Config file object&依赖 -Science Pge Config File Writer&PGE run information&依赖 -Science Pge Config File Writer&configuration file&依赖 -Component Interaction&( xsl ) transformation&依赖 -Component Interaction&SciPgeConfigFileWriter&依赖 -We&NASA mission&依赖 -We&control system ( pc&依赖 -system&high throughput job processing&依赖 -mission&File Manager&依赖 -mission&4.1 Orbiting Carbon Observatory Mission&依赖 -OCO&four terabyte&依赖 -four terabyte&) datum ( ft&AGGREGATION -100 %&datum&AGGREGATION -OCO&PCS software&依赖 -OCO&) datum&依赖 -variant&scientific software&AGGREGATION -variant&science computing facility&依赖 -variant&science computing facility&依赖 -pc&science computing facility&依赖 -gigabyte&iasi datum (&AGGREGATION -hundred&gigabyte&AGGREGATION -4.2 NPP Sounder PEATE Mission Specifically NPP Sounder PEATE&File Manager and Workflow Manager&依赖 -PCS framework&addition&依赖 -PCS framework&4.3 Further application&依赖 -PCS framework&two aforementioned NASA mission&依赖 -PCS framework&’s early detection research network ( edrn ) [ 6 ]&依赖 -work&Network& -JPL&EDRN&依赖 -PCS framework&collection , annotation and dissemination&依赖 -PCS framework&raw scientific datum&依赖 -PCS framework&early detection&依赖 -collection , annotation and dissemination&raw scientific datum&AGGREGATION -early detection&cancer&AGGREGATION -JPL&informatics effort&依赖 -PCS framework&cancer&依赖 -NASA mission&mission&GENERALIZATION -software integration risk&mission development [ 13 ]&依赖 -design phase&mission&AGGREGATION -norm&scratch&依赖 -their&system& -norm&scratch&依赖 -common science data system service&benefit&依赖 -we&standards-based software framework&依赖 -we&end&依赖 -common science data system service&reuse&依赖 -adaptable&requirement&依赖 -benefit&reuse&AGGREGATION -reusable software&most basic science data system&依赖 -most basic science data system&metada management&依赖 -Additional framework&core capability&依赖 -we&paper&依赖 -mission&nasas decadal survey present additional challenge&依赖 -mission&nasas decadal survey present additional challenge&依赖 -mission&orbiting carbon observatory ( oco ) and npp sounder peate mission&依赖 -mission&nasas decadal survey present additional challenge&依赖 -pc&orbiting carbon observatory ( oco ) and npp sounder peate mission&依赖 -mission&orbiting carbon observatory ( oco ) and npp sounder peate mission&依赖 -mission&orbiting carbon observatory ( oco ) and npp sounder peate mission&依赖 -jpl-led soil moisture active passive ( smap ) mission (&operation&依赖 -jpl-led soil moisture active passive ( smap ) mission (&pc&依赖 -jpl-led soil moisture active passive ( smap ) mission (&operation&依赖 -jpl-led soil moisture active passive ( smap ) mission (&pc&依赖 -most earth science datum system&algorithm team&依赖 -most earth science datum system&algorithm team&依赖 -most earth science datum system&operational infrastructure&依赖 -most costly and risky aspect&most earth science datum system&AGGREGATION -most earth science datum system&operational infrastructure&依赖 -cost and risk&development-to-operation&AGGREGATION -our&focus& -easy integration&algorithm&AGGREGATION -JPL-led mission&SMAP&依赖 -Dynamics&ice ( desdynus ) mission&AGGREGATION -challenge&desdynus&AGGREGATION -challenge&deployment&依赖 -challenge&deployment&依赖 -challenge&us&依赖 -deployment&PCS component&AGGREGATION -challenge&PCS component&依赖 -challenge&us&依赖 -challenge&PCS component&依赖 -Acknowledgements This effort&Jet Propulsion Laboratory&依赖 -California Institute&Technology&AGGREGATION -reference&1 ] open archive initiative , http://www.openarchives.org&依赖 -technical report rfc 2396 , 1998&technical report rfc 2396 , 1998&依赖 -a distribute information service architecture&early detection&依赖 -a distribute information service architecture&cancer&依赖 -a distribute information service architecture&a distribute information service architecture&依赖 -a distribute information service architecture&cancer&依赖 -a distribute information service architecture&biomarker discovery&依赖 -a distribute information service architecture&early detection&依赖 -a distribute information service architecture&cancer&依赖 -a distribute information service architecture&a distribute information service architecture&依赖 -a distribute information service architecture&a distribute information service architecture&依赖 -a distribute information service architecture&cancer&依赖 -a distribute information service architecture&a distribute information service architecture&依赖 -a distribute information service architecture&biomarker discovery&依赖 -a distribute information service architecture&early detection&依赖 -a distribute information service architecture&early detection&依赖 -a distribute information service architecture&biomarker discovery&依赖 -a distribute information service architecture&biomarker discovery&依赖 -anatomy&grid&AGGREGATION -Condor-a hunter&idle workstation&AGGREGATION -a software architecture-based framework&a software architecture-based framework&依赖 -a software architecture-based framework&a software architecture-based framework&依赖 -a software architecture-based framework&highly distribute and datum intensive scientific application&依赖 -a software architecture-based framework&a software architecture-based framework&依赖 -a software architecture-based framework&highly distribute and datum intensive scientific application&依赖 -a software architecture-based framework&highly distribute and datum intensive scientific application&依赖 -a software architecture-based framework&highly distribute and datum intensive scientific application&依赖 -a software architecture-based framework&a software architecture-based framework&依赖 -page&ICSE&依赖 -a science datum system approach&smap mission&依赖 -smap mission&mission&GENERALIZATION -a science datum system approach&smap mission&依赖 -taxonomy&apr 2005&依赖 -taxonomy&apr 2005&依赖 -taxonomy&workflow management system&AGGREGATION -taxonomy&apr 2005&依赖 -taxonomy&apr 2005&依赖 diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/A Reusable Process Control System Framework for the Orbiting Carbon Observatory and NPP Sounder PEATE missions.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/A Reusable Process Control System Framework for the Orbiting Carbon Observatory and NPP Sounder PEATE missions.txt deleted file mode 100644 index de3ecb351d1cf7a834e774d043c5e3cc5845f0eb..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/doc/Apache OODT File Manager/A Reusable Process Control System Framework for the Orbiting Carbon Observatory and NPP Sounder PEATE missions.txt +++ /dev/null @@ -1,716 +0,0 @@ -See discussions, stats, and author profiles for this publication at: https://www.researchgate.net/publication/232619682 -A Reusable Process Control System Framework for the Orbiting Carbon -Observatory and NPP Sounder PEATE missions -Article · July 2009 -DOI: 10.1109/SMC-IT.2009.27 -CITATIONS -21 -READS -90 -11 authors, including: -Some of the authors of this publication are also working on these related projects: -Airborne Snow Observatory View project -The Planetary Data System PDS4 Information Model-Driven Architecture View project -Daniel J. Crichton -NASA -138 PUBLICATIONS 791 CITATIONS -SEE PROFILE -Sean Hardman -NASA -20 PUBLICATIONS 53 CITATIONS -SEE PROFILE -Paul Ramirez -NASA -39 PUBLICATIONS 408 CITATIONS -SEE PROFILE -Sean Colin-Patrick Kelly -NASA -43 PUBLICATIONS 213 CITATIONS -SEE PROFILE -All content following this page was uploaded by Paul Ramirez on 22 May 2014. -The user has requested enhancement of the downloaded file. -A Reusable Process Control System Framework for the Orbiting Carbon -Observatory and NPP Sounder PEATE missions -Chris A. Mattmann, Dana Freeborn, Dan Crichton, Brian Foster, -Andrew Hart, David Woollard, Sean Hardman, Paul Ramirez, -Sean Kelly, Albert Y. Chang, Charles E. Miller -Jet Propulsion Laboratory -California Institute of Technology -Pasadena, CA 91109, USA -mattmann@jpl.nasa.gov -Abstract -We describe a reusable architecture and implementation -framework for managing science processing pipelines for -mission ground data systems. Our system, dubbed “PCS”, -for Process Control System, improves upon an existing software -component, the OODT Catalog and Archive (CAS), -which has already supported the QuikSCAT, SeaWinds and -AMT earth science missions. This paper focuses on PCS -within the context of two current earth science missions: the -Orbiting Carbon Observatory (OCO), and NPP Sounder -PEATE projects. -1 Introduction -Data volume and computational needs for Earth science -missions at NASA are growing by orders of magnitude. The -low cost of disk storage space and the increasing power -and pervasiveness of high performance computing have engendered -an era in which previously unimaginable science -questions can be answered in years rather than decades. -These science questions range from the study of sea surface -temperatures to observe maritime pollution, to measuring -atmospheric chemical composition for weather forecasting, -to obtaining a better understanding of the Earth’s global carbon -cycle and climate change as a whole. -A significant portion of any space-based NASA earth -science mission is a Ground Data System (GDS). The GDS -is responsible for receiving raw spacecraft data as delivered -from a ground station1, and processing the information -through several focused series of steps with the goal of -1A strategically placed data center on Earth with ample ground-tospace -bandwidth and connectivity for receiving satellite data. -delivering the scientific value encoded in the data to interested -scientists, both locally at an instrument team center, -and then to universities, decision makers, and the broader -science community. The processing that a GDS must perform -ranges from mundane activities including data (un- -)marshalling (removal of special space “header” information), -and subsetting, to more involved processes including -temporal and spatial positioning, calibration, and statistical -analysis, to complex scientific assimilation including -prospective and retrospective physical modeling of a scene. -Beginning with Automated Multi-Mission Operations -System (AMMOS) Multi-mission Ground Data System -(MGDS) in the early 1990s, our work has focused on building -reusable software components for GDS systems. As -an example, the Central Data Base (CDB) Subsystem of the -MGDS included data base management software comprised -of metadata and file management, file transfer capabilities, -user interfaces and data storage facilities to support multimission -telemetry data streams for current and future planetary -missions. This demanded that the CDB architecture -adhere to the architectural principles of extensibility, scalability, -and reusability. Because the CDB was and is part of -a larger system that included controlled, centralized hardware, -these architectural principles of CDB were satisfied -for AMMOS by simply ensuring that the CDB was data and -policy driven. -Our ensuing work on the Alaska SAR Facility (ASF) and -NASA Scatterometer (NSCAT) projects, made clear two -significant trends: 1) neither of these missions were part -of the controlled, centralized system for which the CDB -was developed and 2) the data management requirements -for these two missions were different from each other and -AMMOS. This meant that 1) hardware and platform choices -could not be assumed and 2) additional capabilities not originally -required for AMMOS had to be developed. In order -to meet mission schedule and cost constraints, developers -for each project independently employed a method we -coined “rapid adaptation” of the original CDB software that -resulted in two very successful mission data systems with -ultimately very few similarities or shared code. -At the time the NSCAT follow-on mission (SeaWinds on -ADEOS II) was ramping up, a technology task originally -funded by the NASA Office of Space Science was focused -on architecting and developing a common, standards-based -software framework dubbed Object Oriented Data Technology -(OODT) [12]. OODT provided “out of the box” core -data management software services while remaining adaptable -to address the (potentially evolving) requirements that -are unique from mission to mission. -Several authors of this paper supporting SeaWinds and -the OODT technology task decided to collaborate to create -a platform- and database-independent service for managing -files and tasks. The result of this collaboration was -the OODT Catalog and Archive Service component that -was architected to be reusable, reliable and scalable. The -SeaWinds (on QuikSCAT and ADEOS II) and Advanced -Communications Technology Satellite (ACTS) Mobile Terminal -(AMT) projects benefited greatly from employing -the CAS component to support their science data systems. -QuikSCAT is in its 10th year of a planned 2-year mission -and is continuing to function in a nearly lights out mode. -Hardware has been added to the system to support the unplanned -data and processing volumes (reprocessing of 7 -years of data completed in 6 months, simultaneous with -daily operations) by simply changing the software configuration. -No software engineers were required to extend the -system. -While the CAS component successfully supported Sea- -Winds and AMT, the following JPL earth missions, Orbiting -Carbon Observatory (OCO) and NPP Sounder PEATE, -needed to support far more complex processing (greatly increased -data volumes and processing throughput) and various -hardware and platform configurations. This forced us to -rethink the CAS component implementation which resulted -in 1) the refactoring of the CAS component into two distinct -components, the File Manager and the Workflow Manager -and 2) the development of a third component to provide a -standard interface to various hardware and platform configurations, -the Resource Manager. -The refactoring of the CAS into the File Manager and the -Workflow Manager components solved several issues. First, -it decoupled the initiation of a workflow from the ingestion -of a file. Therefore, while workflows can be initiated based -on the ingestion of a particular file or file type, they can also -be initiated based on other events such as a specific time of -day, an operator request or a software request. Second, the -refactoring provides developers and system designers the -ability to utilize only the components they need. And third, -the refactoring supports independent evolution of the components, -and thus capabilities. The combination of these -three refactored CAS components have come to be known -as the Process Control System, or PCS. -In addition to the File Manager, Workflow Manager -and Resource Manager components that provide common -reusable capabilities for file and metadata management, -pipeline processing and job submission, we have also developed -reusable interfaces to these components to provide -additional commonly required capabilities for science data -management systems. To support the automation of file ingestion, -we have developed a configurable push-pull framework -and crawler framework. To provide easy integration -of science code in order to support all phases of algorithm -development (testbed, operations and science computing facility), -the PCS Task Wrapper has been developed. -In this paper we will describe our core PCS components, -their architecture, how they helped us solve problems on -OCO and NPP Sounder PEATE, and how they are positioning -us for the future of Earth science mission work. We believe -such work will necessitate the same spirt of architectural -reuse, understanding and mission specific adaptation -that led to the genesis of the modern PCS and that will ultimately -lead to its future evolution. We will argue in this paper -that our PCS uniquely positions us in the state of the art -in constructing large-scale, distributed, data-intensive GDS -software for NASA Earth science missions. -The rest of this paper is organized as follows. Section 2 -provides further background and related efforts in the areas -of grid computing, workflow systems and science data systems. -Section 3 describes the core PCS architectural components -in greater detail. Section 4 presents our experience -leveraging the PCS on OCO and NPP Sounder PEATE. Section -5 rounds out the paper with conclusions and highlights -our planned future work. -2 Background and Related Work -Since the development of the computational grid [8] as -a means for the virtualization and sharing of processing -and storage resources across organizational and geographic -boundaries, many groups and organizations have recognized -the power of the grid as an enabler of large-scale scientific -research. In this paper, we discuss ongoing software -projects and research initiatives relevant to the PCS. -2.1 Grid Systems -The Globus toolkit [9], developed by The Globus Alliance, -is a collection of open-source software tools for developing -distributed computing systems and applications. -The toolkit provides users with a suite of software components -and libraries that can either be used individually or -packaged together to implement the many aspects of a distributed, -service-oriented infrastructure including security, -resource and data discovery, access, and management, and -communication modules customized for a particular gridbased -effort. -2.2 Workflow Systems -The past ten years have witnessed an explosion in the -number of workflow languages and software systems developed -to support scientific workflows. Yu and Buyya [15] -attempted to taxonomize these scientific workflow systems, -largely according the underlying technologies with which -they were built. In addition to this taxonomy, Woollard, et. -al., presented a characterization of workflow systems based -the intended scientific use [14]. Specifically, the authors -classified certain workflow systems as Production Systems, -of which both the OCO and NPP Sounder PEATE ground -data systems are examples. -2.2.1 Condor -Condor [11] is a grid-based job scheduling system developed -at the University of Wisconsin Madison which aims, -among other things, to improve the effective usage of available -distributed computing and storage resources by detecting -and exploiting machine idle cycles. Condor provides -mechanisms for job queuing, setting scheduling policies, -and general resource management and monitoring. Condor -insulates users from the particulars of the details of the underlying -infrastructure by transparently handling decisions -about when and where jobs will be scheduled, monitoring -their execution, and producing notifications of completion. -While originally designed to operate in a workstation environment, -a variant of Condor, Condor-G [10], leverages the -Globus toolkit to provide a Condor implementation that is -interoperable with Globus-based grids. -2.2.2 Pegasus -Pegasus [7] is similar to Condor in that it provides a layer of -abstraction between the jobs to be processed and the hardware -that they will eventually be processed on. Developed -at the USC Information Science Pegasus is capable of dynamically -assigning computational workflows with multiple -processing steps to a large number of grid-based compute -nodes based on resource availability. In addition to generating -an initial workflow mapping, Pegasus offers the ability -to transparently remap a workflow, increasing the reliability -of the system in the event of failure in a small number of -compute nodes. -2.3 Science Data Processing Systems -Science Data Processing Systems provide the base level -of service needed to effectively manage the vast quantities -of intermediate and final data products generated by largescale, -computationally intensive research tasks. While there -are a large number of systems in operation, we focus our -discussion on those which provide services distinctly similar -to the PCS. -2.3.1 S4PA -The Simple, Scalable, Script-based Science Product -Archive (S4PA) [3], is a storage architecture developed and -deployed at NASAs Goddard Space Flight Center in support -of the operation of the Goddard Earth Science Data -and Information Services Center (GES DISC). As cost was -a primary factor in the development of S4PA, the developers -have taken pains to streamline the system. Hosting the -primary copy of all data online reduced the need for costly -physical media distribution, and utilizing the UNIX directory -structure, in combination with metadata-encoded filenames, -provides a simplified mechanism for archive and retrieval. -As its name implies, the S4PA is primarily a data archive -service. The PCS, as described in this paper, addresses data -archiving, but takes a more architecturally grounded approach, -eschewing scripts in favor of first-class architectural -components and connectors to implement complete, endto- -end data processing pipelines. Furthermore, as complete -science data processing pipelines are composed of a large -number of complimentary, interconnected services, a formal -architectural underpinning helps to provide unity and -cohesion among the constituent components. -2.4 Standards -Grid-based science data processing systems have matured -sufficiently for common themes, lessons, and challenges -to emerge among the many participants. As a result, -there are several ongoing efforts to codify the shared knowledge -and experience into formal standards. We discuss the -Open Grid Framework and the Open Archives Initiatives -Protocol for Metadata Harvesting. -2.4.1 OGF -The Open Grid Forum [2] is actively developing standards -and specifications with the goal of spreading the adoption -of grid-based software systems. The OGF is comprised -of business, government, scientific, and academic organizations -and focuses on interoperability as the key to expanding -the utilization of grids. Through both advocacy and policy, -the OGF represents an independent voice on the role of -grids, and their potential to aid modern research. -2.4.2 OAI -The Open Archives Initiative [1] also promotes standards -for interoperability and has developed, among others, the -Protocol for Metadata Harvesting (OMI-PMH). The goal -of the OMI-PMH is to improve application interoperability -by enabling consistency in the way metadata (data about -data) is exposed, accessed, and interpreted. By providing -a flexible, extensible standard interface to the rich array -of application-specific metadata currently stored in nonuniform, -distributed repositories, the OAI hopes to facilitate -the broader accessibility and usability of distributed data resources. -3 PCS Core Architecture -In this section, we describe the PCS core components. -The three PCS manager components, File Manager, Workflow -Manager, and Resource Manager, are daemon-like web -service components responsible for answering basic questions -regarding file locations, metadata, task control and -data flow, and resource availability, monitoring, and usage. -The three PCS frameworks together implement one of -two critical higher level services in data processing systems: -(1) managing the ingestion and acquisition of remotely acquired -datasets, handled via the Crawler Framework and -Push Pull components ; and (2) managing pipeline processing, -product ingestion and data production, handled via the -PCS Task Wrapper. We will describe each component in -greater detail below. The overall PCS architecture described -in this architecture is given in Fig. 1. -3.1 File Manager -The File Manager component is responsible for tracking, -ingesting and moving file data and metadata between a -client system and a server system. The File Manager is an -extensible software component that provides an XML-RPC -external interface, and a fully tailorable Java-based API for -file management. The critical objects managed by the File -Manager include: -Products - Collections of one or more files, and their associated -Metadata. -Metadata - A map of key to multiple values of descriptive -information about a Product. -References - Pointers to a Product file’s original location, -and to its final resting location within the archive constructed -by the File Manager. -Product Type - Descriptive information about a Product -that includes what type of file Uniform Resource Identifier -(URI) [5] generation scheme to use, the root -repository location for a particular Product, and a description -of the Product. -Element - A singular Metadata element, such as “Author”, -or “Creator”. Elements may have additional metadata, -in the form of the associated definition and even a corresponding -Dublin Core [4] attribute. -Versioner - A URI generation scheme for Product Types -that defines the location within the archive (built by -the File Manager) where a file belonging to a Product -(that belongs to the associated Product Type) should be -placed. -Each Product contains one or more References, and one -Metadata object. Each Product is a member of a single -Product Type. The Metadata collected for each Product is -defined by a mapping of Product Type to one or more Elements. -Each Product Type has an associated Versioner. -3.2 Workflow Manager -TheWorkflow Manager component is responsible for description, -execution, and monitoring of Workflows, using a -client, and a server system. Workflows are typically considered -to be sequences of tasks, joined together by control -flow, and data flow, that must execute in some ordered -fashion. Workflows typically generate output data, perform -routine management tasks (such as email, etc.), or describe -a business’s internal routine practices [14]. The Workflow -Manager is an extensible software component that provides -an XML-RPC external interface, and a fully tailorable Javabased -API for workflow management. The critical objects -managed by the Workflow Manager include: -Events - are what triggerWorkflows to be executed. Events -are named, and contain dynamic Metadata information, -passed in by the user. -Metadata - a dynamic set of properties, and values, provided -to a WorkflowInstance via a user-triggered -Event. -Workflow - a description of both the control flow, and data -flow of a sequence of tasks (or stages that must be executed -in some order. -Workflow Instance - an instance of a Workflow, typically -containing additional runtime descriptive information, -such as start time, end time, task wall clock time, etc. -A WorkflowInstance also contains a shared Metadata -context, passed in by the user who triggered theWorkflow. -This context can be read/written to by the underlying -WorkflowTasks, present in a Workflow. -Workflow Tasks - descriptions of data flow, and an underlying -process, or stage, that is part of a Workflow. -Workflow Task Instances - the actual executing code, or -process, that performs the work in the Workflow Task. -Workflow Task Configuration - static configuration -properties, that configure a WorkflowTask. -Workflow Conditions - any pre (or post) conditions on the -execution of a WorkflowTask. -Workflow Condition Instances - the actual executing -code, or process, that performs the work in the Workflow -Condition. -Each Event initiates one or more Workflow Instances, -providing a Metadata context (submitted by an external -user). Each Workflow Instance is a run-time execution -model of a Workflow. Each Workflow contains one or -more Workflow Tasks. Each Workflow Task contains a single -Workflow Task Configuration, and one or more Workflow -Conditions. Each Workflow Task has a corresponding -Workflow Task Instance (that it models), as does each -Workflow Condition have a corresponding Workflow Condition -Instance. -3.3 Resource Manager -The Resource Manager component is responsible for excecution, -monitoring and traacking of jobs, storage and networking -resources for an underlying set of hardware resources. -The Resource Manager is an extensible software -component that provides an XML-RPC external interface, -and a fully tailorable Java-based API for resource management. -The critical objects managed by the Resource Manager -include: -Job - an abstract representation of an execution unit, that -stores information about an underlying program, or execution -that must be run on some hardware node ,including -information about the Job Input that the Job -requires, information about the job load, and the queue -that the job should be submitted to. -Job Input - an abstrct representation of the input that a Job -requires. -Job Spec - a complete specification of a Job, including its -Job Input, and the Job definition itself. -Job Instance - the physical code that performs the underlying -job execution. -Resource Node - an available execution node that a Job is -sent to by the Resource Manager. -Each Job Spec contains exactly one Job, and Job Input. -Each Job Input is provided to a single Job. Each Job describes -a single Job Instance. And finally, each Job is sent -to exactly one Resource Node. -3.4 Crawler Framework -The Crawler Framework was an effort to standardize the -common ingestion activities that occur both in data acquisition -and archival, as well as those that occur in pipeline -processing. These types of activities regularly involve identification -of files and directories to crawl (based on e.g., -mime type, regular expressions, or direct user input), satisfaction -of ingestion pre-conditions (e.g., the current crawled -file has not been previously ingested), followed by metadata -extraction. After metadata extraction, crawled data follows -a standard three state lifecycle: (1) preIngestion - where -e.g., a file may be unzipped or pre-processed prior to ingestion; -(2) postIngest success, indicating a successful ingestion -has occurred and e.g., the origin data file from the -ingest area should be deleted; and (3) postIngest failure, indicating -that ingestion was not successful and some corrective -action, e.g,. moving the failed file to a failure area for -later examination, should occur. -To date, we have identified three types of Product -Crawlers, where each Crawler varies along the lines of customized -precondition verification, crawilng strategy, and -need for metadata extraction. The StdProductCrawler assumes -that a Metadata object has already been generated -and included with a Product prior to ingestion, so no further -work is required to generate Metadata from a Product – -the Product is ready to be ingested. The MetExtractorProductCrawler -is responsible for generating a Metadata object -dynamically, as files are encountered during the crawling -process. Finally, the AutoDetectCrawler uses a content -type identification and regular-expressions to identify Product -Types dynamically, and then defaults to the behavior of -the MetExtractorProductCrawler for Product Types identified -via content detection. The critical objects managed by -the Crawler Framework are: -Crawler Action - is attached to one or more of the three -phases, and when a ProductCrawler enters a given -phases, all the CrawlerActions attached to that phase -are executed. The valid phases are: preIngest, -postIngestSuccess and postIngestFailure. -Precondition Comparator - is used by MetExtractorProductCrawler -and AutoDetectProductCrawler. They are -part of those ProductCrawlers customized implementation -of precondition verification that identify appropriate -times to stifle or allow metadata extractor and -ultimately ingestion, to occur. -Metadata Extractor - is run by the MetExtractorProductCrawler -and the AutoDetectProductCrawler to -generate Metadata for a Product file based on some -business rules and logic. -3.5 Push Pull Framework -The Crawler Framework supports many generic ingestion -services, including metadata extraction, crawling, and -ingestion, however, one service that necessitated further -work was the development of a protocol layer allowing -a ProductCrawler to obtain content using protocol plugins -that download content using implementations of remote -protocols such as HTTP, FTP, WinNT file system, HTTPS, -etc. -The Push Pull Framework is responsible for remote data -acquisition and acceptance over modern web protocols, -such as those mentioned above. The Push Pull Framework -is flexible in that it provides the ability to plug in different -Metadata Extractors, Data Protocols, Content Types, -etc. The framework supports parallel file transfers and data -downloads, email-based push data acceptance using IMAP, -SMTP protocols, and the ability to configure “Virtual” remote -directories (based on Metadata such as Date/Time) -from which files can be downloaded. -The critical objects managed by the Push Pull Framework -are: -Retrieval Method - defines the manner in which files are -retrieved from remote sites. It is given a configuration -file, a the Parser for the file, and a FileRetrievalSystem -(which handles all the complexities of multi-threaded -file downloading). There are currently two out-of-thebox -RetrievalMethods: RemoteCrawler and ListRetriever. -RemoteCrawler is a configurable remote site -directory and file regular expression filterable crawler. -ListRetriever will download a given list of file URIs -[5]. -Parser - parses a given configuration file into a Virtual- -FileStructure which is use to filter URIs to download. -Protocol - handles file transfer and communication via -some transfer protocol. Currently implemented Protocols -include: sftp, ftp, http, imaps, file (localhost). -3.6 PCS Task Wrapper -The PCS Task Wrapper framework is responsible for -standardizing the setup, process initiation, execution and -file management tasks surrounding execution of NASA -Product Generation Executives, or PGEs. PGEs codify a -scientific algorithm, some step in the overall scientific process -involved in a mission science workflow. -The PCS Task Wrapper provides a stable operating environment -to the underlying PGE during its execution lifecycle. -If the PGE requires a file, or metadata regarding the -file, the PCS Task Wrapper is responsible for delivering that -information to the PGE in a manner that meets its requirements. -If the PGE requires knowledge of upstream or downstream -PGEs in a sequence of executions, that information -is also made available, and finally if information regarding -disk space, node information such as CPU availability, etc., -is required the PCS Task Wrapper provides this information -to the underlying PGE. After this information is collected, -the PGE is executed and its output Product file and -Metadata generation is managed via the PCS Task Wrapper -framework. The PCS Task Wrapper is responsible for -marshalling output Products and Metadata back to the File -Manager for use in downstream data processing and pedigree. -In support of this, the PCS Task Wrapper leverages -the Crawler Framework to ingest (during pipeline processing) -the output Product files and Metadata produced by the -PGE. -As can be gleaned from the above discussion, the PGE -Task Wrapper is really the unifying bridge between the execution -of a step in the overall processing pipeline, and the -available PCS component services and the information that -they collectively manage. -The critical objects managed by the PCS Task Wrapper -are: -PGETaskInstance - an abstract class which contains a -generalized set of actions usually performed when running -PGEs. Every variable and method is protected, -thus allowing subclasses to easily modify just those -generalized actions which need to be customized for -different PGE. -Pge Config File Builder - builds a PgeConfig object and -set additional Metadata which codifies the information -necessary for orchestrating a PGE through its lifecycle. -The PCS Task Wrapper is based on a simple but -powerful XML syntax which allows a scientist to simply -fill out an xml file to describe the necessary steps -to execute a PGE. -Config File Property Adder - builds the Pge Config file -object and sets custom PGE Metadata. This allows -for a general PgeConfigBuilder with different Config- -FilePropertyAdders for setting PGE specific fields in -the PgeConfig object. -Science Pge Config File Writer - passes a PGE run information -via configuration files. This object allows -for any number of config files in any desired format -to be generated describing PGE input and those files -to be delivered to the PGE. The PCS Task Wrapper -provides existing implementations, including a deFigure -1. Component Interaction Within the PCS -fault XML Stylesheet Language (XSL) Transformation -based SciPgeConfigFileWriter. -Pcs Met File Writer - aids in generating Metadata objects -associated with PGE output products. -4 Experience and Evaluation -We have successfully applied the Process Control System -(PCS) to existing NASA missions: the Orbiting Carbon -Observatory (OCO) mission, and the NPP Sounder PEATE -mission. Both systems involve tasks such as high throughput -job processing, terabyte-scale data management, and -science computing facilities. -4.1 Orbiting Carbon Observatory Mission -On OCO, the mission is using the File Manager to ingest -MODIS, CloudSat and other ancillary data products -for use in the high performance Level 2 Science Algorithm. -To date, OCO has already used the PCS software to process -over four terabytes of Fourier Transform Spectrometer -(FTS) data provided by ground-based instruments located -around the country (e.g., Park falls, Montana, and Darwin, -Australia), and has used the software to support Instrument -Thermal Vacuum (TVAC) testing, processing 100% of all -data taken by the OCO instrument during TVAC. Also, the -PCS supports a science computing facility in which variants -of scientific software can be excursive prior to inclusion in -an operations Pipeline. -4.2 NPP Sounder PEATE Mission -Specifically NPP Sounder PEATE has already used the -File Manager and Workflow Manager to ingest and process -hundreds of gigabytes of IASI data (and is in preparation to -accept CRIMS data). Also on PEATE, the PCS is currently -being used to re-catalog over fifteen million existing science -data products from the NASA AIRS missions TLSCF. -In addition, the Resource Manager will be used on NPP to -support job processing across an eighty-node cluster. -4.3 Further Applications -In addition to the two aforementioned NASA missions, -the PCS framework is being leveraged on reimbursable -work for the National Cancer Institute (NCI)’s Early Detection -Research Network (EDRN) [6]. JPL leads the informatics -efforts on EDRN, and the PCS framework is being -used in the collection, annotation and dissemination of raw -scientific data supporting the early detection of cancer to -scientists across the country. -In the next year, PCS will also be used to support a new -JPL-led NASA mission, the Soil Moisture Active Passive -(SMAP) mission. The science computing facility designs -on OCO and NPP have been used to create an algorithm -testbed for SMAP scientists early in the design phase of the -mission so that software integration risks can be mitigated -during mission development [13]. -5 Conclusions and Future Work -While the norm for earth science missions has been for -each mission to develop their own one-off science data system -from scratch, the continual decrease in mission funding -combined with the exponential increase in mission complexity -(data volume and processing throughput) over the -last decade has made this approach pass´e and risky. It was -clear that the need for a new approach was eminent. -To this end, we have developed a standards-based software -framework to provide common science data system -services that yields the benefits of reuse while remaining -adaptable to address the requirements that are unique to the -customer. This reusable software is centered around the -most basic science data system functions that support file -and metadata management, workflow management, and resource -management. Additional frameworks augment the -core capabilities to provide automation for remote data acquisition, -data ingestion and standard pipeline processing. -This reusable software framework is the Process Control -System (PCS) we have described in this paper. -While the PCS has successfully supported the Orbiting -Carbon Observatory (OCO) and NPP Sounder PEATE -missions, upcoming missions in NASAs Decadal Survey -present additional challenges. The JPL-led Soil Moisture -Active Passive (SMAP) Mission (currently in formulation -phase) will be using the PCS not only for operations, but -also for the algorithm testbed and the science computing facility. -Providing the operational infrastructure to the algorithm -team early in the mission lifecycle will greatly reduce -the cost and risk of development-to-operations for the most -costly and risky aspect of most earth science data systems, -the algorithms. However, this also means that easy integration -of algorithms and dynamic workflow specification -are our current focus for extending the PCS capabilities. -Not far behind SMAP is another JPL-led mission, Deformation, -Ecosystem Structure and Dynamics of Ice (DESDynI) -Mission. The challenges of DESDynI are requiring -us to consider the deployment of PCS components to support -a grid architecture, supporting distributed file management -and processing capabilities supported by centralized -access to a virtual science data system. -Acknowledgements -This effort was supported by the Jet Propulsion Laboratory, -managed by the California Institute of Technology -under a contract with the National Aeronautics and Space -Administration. -References -[1] Open archives initiative, http://www.openarchives.org. -[2] Open grid forum, http://www.ogf.org. -[3] S4pa, http://daac.gsfc.nasa.gov/techlab/s4pa/index.shtml. -[4] Dublin core metadata element set, 1999. -[5] T. Berners-Lee, R. Fielding, and L. Masinter. Uniform resource -identifiers (uri): Generic syntax. Technical Report -RFC 2396, 1998. -[6] D. Crichton, S. Kelly, C. Mattmann, Q. Xiao, J. S. Hughes, -J. Oh, M. Thornquist, D. Johnsey, S. Srivastava, L. Essermann, -and W. Bigbee. A distributed information services -architecture to support biomarker discovery in early detection -of cancer. In e-Science, page 44, 2006. -[7] E. Deelman, J. Blythe, Y. Gil, C. Kesselman, G. Mehta, -S. Patil, M.-H. Su, K. Vahi, and M. Livny. Pegasus: Mapping -Scientific Workflows onto the Grid. 2004. -[8] I. Foster. The anatomy of the grid: Enabling scalable virtual -organizations. pages 6–7, 2001. -[9] I. Foster. Globus toolkit version 4: Software for serviceoriented -systems. pages 2–13. 2005. -[10] J. Frey, T. Tannenbaum, M. Livny, I. Foster, and S. Tuecke. -Condor-g: A computation management agent for multiinstitutional -grids. Cluster Computing, 5(3):237–246, July -2002. -[11] M. J. Litzkow, M. Livny, and M.W. Mutka. Condor-a hunter -of idle workstations. pages 104–111, 1988. -[12] C. Mattmann, D. J. Crichton, N. Medvidovic, and -S. Hughes. A software architecture-based framework for -highly distributed and data intensive scientific applications. -In ICSE, pages 721–730, 2006. -[13] D. Woollard, O. ig Kwoun, T. Bicknell, S. Dunbar, and -K. Leung. A science data system approach for the smap -mission. In IEEE Radar, 2009. -[14] D. Woollard, N. Medvidovic, Y. Gil, and C. A. Mattmann. -Scientific software as workflows: From discovery to distribution. -Software, IEEE, 25(4):37–43, 2008. -[15] J. Yu and R. Buyya. A taxonomy of workflow management -systems for grid computing, Apr 2005. -View publication stats \ No newline at end of file diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/A Reusable Process Control System Framework for the Orbiting Carbon Observatory and NPP Sounder PEATE missions.txt.xml.xls b/src/main/resources/sdtocode/doc/Apache OODT File Manager/A Reusable Process Control System Framework for the Orbiting Carbon Observatory and NPP Sounder PEATE missions.txt.xml.xls deleted file mode 100644 index e51ead435b360e1edf0462697e2b0f8580f0530d..0000000000000000000000000000000000000000 Binary files a/src/main/resources/sdtocode/doc/Apache OODT File Manager/A Reusable Process Control System Framework for the Orbiting Carbon Observatory and NPP Sounder PEATE missions.txt.xml.xls and /dev/null differ diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Catalog and Archive File Management Component-relation.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/Catalog and Archive File Management Component-relation.txt deleted file mode 100644 index 6d6eda2f3f2f79375a3b058102f0978e4df65528..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Catalog and Archive File Management Component-relation.txt +++ /dev/null @@ -1,225 +0,0 @@ -catalog and archive file management component introduction project description architecture extension points current extension point implementations use case conclusion this&Catalog&依赖 -its&points& -guide&tailorable extension point&依赖 -guide&File Manager architecture and interface&依赖 -our&Guides& -remainder§ion&依赖 -remainder&guide&AGGREGATION -we&architecture&依赖 -we&constituent component&依赖 -its&components& -we&architecture&依赖 -we&constituent component&依赖 -architecture&File Manager&AGGREGATION -Client and Server&major component&依赖 -Client and Server&File Manager&依赖 -major component&File Manager&AGGREGATION -relationship&diagram&依赖 -Catalog&metada&依赖 -File Manager Server&Repository&依赖 -Catalog&Validation Layer&依赖 -products&location& -File Manager Server&' location&依赖 -Repository&product&依赖 -File Manager Server&archive as&依赖 -Transfer&Transfer&依赖 -Transfer&Transfer&依赖 -Transfer&data product&AGGREGATION -Transfer&Transfer&依赖 -domain&Transfer&AGGREGATION -Transfer&Transfer&依赖 -Transfer&Transfer&依赖 -Transfer&Transfer&依赖 -collection&one or more file&AGGREGATION -their&Metadata& -A map&key&AGGREGATION -> multiple value&descriptive information&AGGREGATION -See&Metadata&依赖 -See&more information&依赖 -its&location& -File Manager&what type&依赖 -File Manager&file urus generation scheme&依赖 -what type&file urus generation scheme&AGGREGATION -description&Product&AGGREGATION -element&form&依赖 -element&associated definition&依赖 -form&associated definition&AGGREGATION -element&additional metada&依赖 -A URI generation scheme&location&依赖 -A URI generation scheme&built&依赖 -A URI generation scheme&archive (&依赖 -Product&1 or more reference&依赖 -member&single Product Type&AGGREGATION -Product&single Product Type&依赖 -Product&mapping&依赖 -Product&Product Type&依赖 -mapping&Product Type&AGGREGATION -Product Type&associated Versioner&依赖 -relationship&below figure&依赖 -new&key capability&AGGREGATION -Easy management&different type&AGGREGATION -different type&Products&AGGREGATION -their&information& -their&ID& -Management&Product Types&AGGREGATION -Management&new type&依赖 -their&name& -Management&new type&依赖 -different kind&back end catalog&AGGREGATION -Catalog extension point&extension point&GENERALIZATION -Catalog extension point&product instance metada and file location information&依赖 -different type&back end data store&AGGREGATION -implementation&end database&依赖 -implementation&JDBC&依赖 -implementation&end database&依赖 -implementation&JDBC&依赖 -implementation&Catalog interface&AGGREGATION -Management&Product instance information&AGGREGATION -Management&include add , delete and update product instance information&依赖 -Management&file location&依赖 -It&Metadata and reference&依赖 -Manager&point& -different type&back end store&AGGREGATION -management&Element policy information&AGGREGATION -Element policy&instance&依赖 -Element policy&XML file&依赖 -Data Transfer&Transfer&GENERALIZATION -File Manager&different Data Transfer protocol&依赖 -Versioner extension point&different File Repository layout&依赖 -Versioner extension point&extension point&GENERALIZATION -Flat product&singular file&依赖 -collection&singular file&AGGREGATION -Products&directory&依赖 -collection&directory&AGGREGATION -File Manager&popular client-server paradigm&依赖 -File Manager&XML-RPC&依赖 -its&interface& -File Manager&File Manager client and server&依赖 -File Manager&main external interface&依赖 -little brother&SOAP&AGGREGATION -File Manager web interface&RSS-based syndication&依赖 -RSS-based syndication&Product feed&AGGREGATION -datum&status tracking&依赖 -RSS-feed&transfer&AGGREGATION -File Manager¤t Product and File transfer&依赖 -Extension Points We&file manager make use&依赖 -file manager make use&factory method pattern&AGGREGATION -interface&many implementation&依赖 -extension point&File Manager&依赖 -different implementation&interface&AGGREGATION -it&software component configuration&依赖 -it&different implementation&依赖 -it&interface&实现 -File Manager extension point&implementation&依赖 -File Manager extension point&two interface&实现 -implementation&two interface&AGGREGATION -File Manager extension point&extension factory&实现 -File Manager load&factory class&依赖 -File Manager load&factory class&依赖 -File Manager load&run-time&依赖 -File Manager load&run-time&依赖 -File Manager&example&依赖 -File Manager&database-based Catalog&依赖 -it&Lucene-based Catalog&依赖 -it&many different type&实现 -The Data Transfer extension point&Product&依赖 -The Data Transfer extension point&movement&依赖 -The Data Transfer extension point&archive&依赖 -movement&Product&AGGREGATION -Different protocol&local ( disk-based ) copy&依赖 -Different protocol&local ( disk-based ) copy&依赖 -extension point&Product Type&依赖 -extension point&element&依赖 -different URI generation scheme&file&依赖 -different URI generation scheme&final resting location&依赖 -final resting location&file&AGGREGATION -different URI generation scheme&particular Product&依赖 -definition&different URI generation scheme&AGGREGATION -Catalog Data Source&base catalog&依赖 -implementation&Catalog extension point interface&AGGREGATION -implementation&a jdbc accessible database backend&依赖 -lucene base catalog&lucene base catalog&依赖 -implementation&catalog extension point interface&AGGREGATION -implementation&catalog extension point interface&依赖 -lucene base catalog&Lucene free text index system&依赖 -implementation&catalog extension point interface&依赖 -implementation&Data Transfer interface&AGGREGATION -Apache&commons-io& -implementation&locally accessible network file system ( nfs ) disk&依赖 -implementation&XML-RPC File Manager client&依赖 -XML-RPC File Manager client&File Manager client&GENERALIZATION -implementation&XML-RPC File Manager client&依赖 -implementation&datum transfer interface&AGGREGATION -InPlace Data Transfer .&product&依赖 -implementation&product type policy information&依赖 -implementation&product type policy information&依赖 -implementation&repository manager extension point&AGGREGATION -implementation&JDBC accessible database&依赖 -implementation&JDBC accessible database&依赖 -XML file&file&GENERALIZATION -implementation&JDBC accessible database&依赖 -implementation&JDBC accessible database&依赖 -implementation&element policy information&依赖 -implementation&element policy information&依赖 -implementation&validation layer extension point&AGGREGATION -Validation Layer extension point&2 XML file&依赖 -implementation&) xml-rpc&依赖 -Validation Layer extension point&Element policy information&依赖 -implementation&( file manager client&依赖 -implementation&) xml-rpc&依赖 -implementation&Validation Layer extension point&AGGREGATION -implementation&( file manager client&依赖 -File Manager&transportation medium&依赖 -implementation&File Manager&依赖 -File Manager&use xml-rpc&依赖 -implementation&File Manager&依赖 -File Manager&use xml-rpc&依赖 -implementation&File Manager&依赖 -File Manager&transportation medium&依赖 -implementation&external server interface&AGGREGATION -XML-RPC&File Manager client&依赖 -implementation&client interface&AGGREGATION -implementation&XML-RPC File Manager server&依赖 -XML-RPC File Manager server&transportation medium&依赖 -implementation&XML-RPC File Manager server&依赖 -XML-RPC File Manager server&XML-RPC&依赖 -implementation&XML-RPC File Manager server&依赖 -several&above capability&AGGREGATION -we&that&依赖 -manager ingest use case red number&step&依赖 -manager ingest use case red number&step&依赖 -sequence&step&AGGREGATION -series&interaction&AGGREGATION -manager ingest use case red number&sequence&依赖 -manager ingest use case red number&sequence&依赖 -ingest operation&ingest&依赖 -File Manager client&Step 1&依赖 -ingest operation&a particular product&依赖 -File Manager client&ingest operation&依赖 -ingest operation&Metadata and References&依赖 -server&point& -System Interface&information&依赖 -it&path& -System Interface&made&依赖 -System Interface&Product Type policy&依赖 -Metadata&Catalog extension point&依赖 -System Interface&file reference&依赖 -Catalog extension point&catalog process&依赖 -Catalog extension point&Validation Layer&依赖 -its&Type& -first step&’s associated versioner&依赖 -Product&Versioner& -first step&’s associated versioner&依赖 -Data Transfer&client or server end&依赖 -aim&document&AGGREGATION -document&architecture&依赖 -document&constituent components , object model and key capability&依赖 -Manager&architecture& -its&model& -current implementation&extension point&AGGREGATION -overview¤t implementation&AGGREGATION -Manager&points& -we&topic&依赖 -we&topic&依赖 -we&installation , configuration , and example use&依赖 -we&installation , configuration , and example use&依赖 diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Catalog and Archive File Management Component.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/Catalog and Archive File Management Component.txt deleted file mode 100644 index 98281efb2fac7059f9bd7452c63995f31d82f4cf..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Catalog and Archive File Management Component.txt +++ /dev/null @@ -1,109 +0,0 @@ -Catalog and Archive File Management Component -Introduction -Project Description -Architecture -Extension Points -Current Extension Point Implementations -Use Cases -Conclusion -This is the developer guide for the Apache OODT Catalog and Archive Service (CAS) File Manager component, or File Manager for short. Primarily, this guide will explain the File Manager architecture and interfaces, including its tailorable extension points. For information on installation, configuration, and examples, please see our User Guides. - -The remainder of this guide is separated into the following sections: - -Project Description -Architecture -Extension Points -Current Extension Point Implementations -Project Description -The File Manager component is responsible for tracking, ingesting and moving file data and metadata between a client system and a server system. The File Manager is an extensible software component that provides an XML-RPC external interface, and a fully tailorable Java-based API for file management. - -Architecture -In this section, we will describe the architecture of the File Manager, including its constituent components, object model, and key capabilities. - -Components -The major components of the File Manager are the Client and Server, the Repository Manager, the Catalog, the Validation Layer, the Versioner, and the Transferer. The relationship between all of these components are shown in the diagram below: - -File Manager Architecture - -The File Manager Server contains both a Repository that manages products (and the products' location in the archive as specified by Versioner), and a Catalog that validates metadata via the Validation Layer. Transfer of data products from the Client to the Server is the domain of the Transfer and can be initiated at either the Client or the Server. - -Object Model -The critical objects managed by the File Manager include: - -Products - Collections of one or more files, and their associated Metadata. -Metadata - A map of key->multiple values of descriptive information about a Product. See CAS-Metadata for more information on Metadata. -Reference - A pointer to a Product file's (or files') original location, and to its final resting location within the archive constructed by the File Manager. -Product Type - Descriptive information about a Product that includes what type of file URI generation scheme to use, the root repository location for a particular Product, and a description of the Product. -Element - A singular Metadata element, such as "Author", or "Creator". Elements may have additional metadata, in the form of the associated definition and even a corresponding Dublin Core attribute. See CAS-Metadata for more information on Metadata Elements. -Versioner - A URI generation scheme for Product Types that defines the location within the archive (built by the File Manager) where a file belonging to a Product (that belongs to the associated Product Type) should be placed. -Each Product contains 1 or more References, and one Metadata object. Each Product is a member of a single Product Type. The Metadata collected for each Product is defined by a mapping of Product Type->1...* Elements. Each Product Type has an associated Versioner. These relationships are shown in the below figure. - -File Manager Object Model -Key Capabilities -The File manager has been designed with a new of key capabilities in mind. These capabilities include: - -Easy management of different types of Products. The Repository Manager extension point is responsible for managing Product Types, and their associated information. Management of Product Types includes adding new types, deleting and updating existing types, and retrieving Product Type Objects, by their ID or by their name. - -Support for different kinds of back end catalogs. The Catalog extension point allows Product instance metadata and file location information to be stored in different types of back end data stores quite easily. Existing implementations of the Catalog interface include a JDBC based back end database, along with a flat-file index powered by Lucene. - -Management of Product instance information. Management includes adding, deleting and updating product instance information, including file locations (References), along with Product Metadata. It also includes retrieving Metadata and References associated with existing Products as well as obtaining the Products themselves. - -Element management for Metadata. The File Manager's Validation Layer extension point allows for the management of Element policy information in different types of back end stores. For instance, Element policy could be stored in XML files, a Database, or a Metadata Registry. - -Data transfer mechanism interface. By having an extension point for Data Transfer, the File Manager can support different Data Transfer protocols, both local and remote. - -Advanced support for File Repository layouts. The Versioner extension point allows for different File Repository layouts based on Product Types. - -Support for multiple Product structures. The File Manager Client allows for Products to be Flat, or Hierarchical-based. Flat products are collections of singular files that are aggregated together to make a Product. Hierarchical Products are Products that contain collections of directories, and sub-directories, and files. - -Design for scalability. The File Manager uses the popular client-server paradigm, allowing new File Manager servers to be instantiated, as needed, without affecting the File Manager clients, and vice-versa. - -Standard communication protocols. The File Manager uses XML-RPC as its main external interface between the File Manager client and server. XML-RPC, the little brother of SOAP, is fast, extensible, and uses the underlying HTTP protocol for data transfer. - -RSS-based Product syndication. The File Manager web interface allows for the RSS-based syndication of Product feeds based on Product Type. - -Data transfer status tracking. The File Manager tracks all current Product and File transfers and even publishes an RSS-feed of existing transfers. - -This capability set is not exhaustive, and is meant to give the user a feel for what general features are provided by the File Manager. Most likely the user will find that the File Manager provides many other capabilities besides those described here. - -Extension Points -We have constructed the File Manager making use of the factory method pattern to provide multiple extension points for the File Manager. An extension point is an interface within the File Manager that can have many implementations. This is particularly useful when it comes to software component configuration because it allows different implementations of an existing interface to be selected at deployment time. - -The factory method pattern is a creational pattern common to object oriented design. Each File Manager extension point involves the implementation of two interfaces: an extension factory and an extension implementation. At run-time, the File Manager loads a properties file specifies a factory class to use during extension point instantiation. For example, the File Manager may communicate with a database-based Catalog and an XML-based Element Store (called a Validation Layer), or it may use a Lucene-based Catalog and a database-based Validation Layer. -Using extension points, it is fairly simple to support many different types of what are typically referred to as "plug-in architectures." Each of the core extension points for the File Manager is described below: - -Catalog The Catalog extension point is responsible for storing all the instance data for Products, Metadata, and for file References. Additionally, the Catalog provides a query capability for Products. -Data Transfer The Data Transfer extension point allows for the movement of a Product to and from the archive managed by the File Manager component. Different protocols for Data Transfer may include local (disk-based) copy, or remote XML-RPC based transfer across networked machines. -Repository Manager The Repository Manager extension point provides a means for managing all of the policy information (i.e., the Product Types and their associated information) for Products managed by the File Manager. -Validation Layer The Validation Layer extension point allows for the querying of element definitions associated with a particular Product Type. The extension point also maps Product Type to Elements. -Versioning The Versioning extension point allows for the definition of different URI generation schemes that define the final resting location of files for a particular Product. -System The extension point that provides the external interface to the File Manager services. This includes the File Manager server interface, as well as the associated File Manager client interface, that communicates with the server. -Current Extension Point Implementations -There are at least two implementations of all of the aforementioned extension points for the File Manager. Each extension point implementation is detailed in this section. - -Catalog -Data Source based Catalog. An implementation of the Catalog extension point interface that uses a JDBC accessible database backend. -Lucene based Catalog. An implementation of the Catalog extension point interface that uses the Lucene free text index system to store Product instance information. -Data Transfer -Local Data Transfer. An implementation of the Data Transfer interface that uses Apache's commons-io to perform local, disk based filesystem data transfer. This implementation also supports locally accessible Network File System (NFS) disks. -Remote Data Transfer. An implementation of the Data Transfer interface that uses the XML-RPC File Manager client to transfer files to a remote XML-RPC File Manager server. -InPlace Data Transfer. An implementation of the Data Transfer interface that avoids transfering any products -- this can be used in the situation where metadata about a particular product should be recorded, but no physical transfer needs to occur. -Repository Manager -Data Source based Repository Manager. An implementation of the Repository Manager extension point that stores Product Type policy information in a JDBC accessible database. -XML based Repository Manager. An implementation of the Repository Manager extension point that stores Product Type policy information in an XML file called product-types.xml -Validation Layer -Data Source based Validation Layer. An implementation of the Validation Layer extension point that stores Element policy information in a JDBC accessible database. -XML based Validation Layer. An implementation of the Validation Layer extension point that stores Element policy information in 2 XML files called elements.xml and product-type-element-map.xml -System (File Manager client and File Manager server) -XML-RPC based File Manager server. An implementation of the external server interface for the File Manager that uses XML-RPC as the transportation medium. -XML-RPC based File Manager client. An implementation of the client interface for the XML-RPC File Manager server that uses XML-RPC as the transportation medium. -Use Cases -The File Manager was built to support several of the above capabilities outlined in Section 3. In particular there were several use cases that we wanted to support, some of which are described below. - -File Manager Ingest Use Case -The red numbers in the above Figure correspond to a sequence of steps that occurs and a series of interactions between the different File Manager extension points in order to perform the file ingestion activity. In Step 1, a File Manager client is invoked for the ingest operation, which sends Metadata and References for a particular Product to ingest to the File Manager server’s System Interface extension point. The System Interface uses the information about Product Type policy made available by the Repository Manager in order to understand whether or not the product should be transferred, where it’s root repository path should be, and so on. The System Interface then catalogs the file References and Metadata using the Catalog extension point. During this catalog process, the Catalog extension point uses the Validation Layer to determine which Elements should be extracted for the particular Product, based upon its Product Type. After that, Data Transfer is initiated either at the client or server end, and the first step to Data Transfer is using the Product’s associated Versioner to generate final file References. After final file References have been determined, the file data is transferred by the server or by the client, using the Data Transfer extension point. - -Conclusion -The aim of this document is to provide information relevant to developers about the CAS File Manager. Specifically, this document has described the File Manager's architecture, including its constituent components, object model and key capabilities. Additionally, the this document provides an overview of the current implementations of the File Manager's extension points. - -In the Basic User Guide and Advanced User Guide, we will cover topics like installation, configuration, and example uses as well as advanced topics like scaling and other tips and tricks. \ No newline at end of file diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Catalog and Archive File Management Component.txt.xml.xls b/src/main/resources/sdtocode/doc/Apache OODT File Manager/Catalog and Archive File Management Component.txt.xml.xls deleted file mode 100644 index 3214e495f6c85d9cd627ada1113d1fba90949502..0000000000000000000000000000000000000000 Binary files a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Catalog and Archive File Management Component.txt.xml.xls and /dev/null differ diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/File Manager Scale Out Planning - OODT - Apache Software Foundation-relation.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/File Manager Scale Out Planning - OODT - Apache Software Foundation-relation.txt deleted file mode 100644 index 5e6d1644140bd63e9c95676ad76a2c09661c8a4e..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/doc/Apache OODT File Manager/File Manager Scale Out Planning - OODT - Apache Software Foundation-relation.txt +++ /dev/null @@ -1,79 +0,0 @@ -seamless querying&multiple filemanager&AGGREGATION -Designer Developers QA Goals Allow&multiple filemanager&依赖 -Designer Developers QA Goals Allow&seamless querying&依赖 -Designer Developers QA Goals Allow&multiple filemanager&依赖 -Designer Developers QA Goals Allow&separate network&依赖 -Designer Developers QA Goals Allow&oodt filemanager linear scalaing&依赖 -Designer Developers QA Goals Allow&seamless querying&依赖 -Designer Developers QA Goals Allow&oodt filemanager linear scalaing&依赖 -Designer Developers QA Goals Allow&separate network&依赖 -Designer Developers QA Goals Allow&oodt filemanager linear scalaing&依赖 -Designer Developers QA Goals Allow&separate network&依赖 -Designer Developers QA Goals Allow&multiple filemanager&依赖 -Designer Developers QA Goals Allow&seamless querying&依赖 -Designer Developers QA Goals Allow&separate network&依赖 -Designer Developers QA Goals Allow&oodt filemanager linear scalaing&依赖 -Designer Developers QA Goals Allow&multiple filemanager&依赖 -Designer Developers QA Goals Allow&multiple filemanager&依赖 -Designer Developers QA Goals Allow&separate network&依赖 -Designer Developers QA Goals Allow&seamless querying&依赖 -Designer Developers QA Goals Allow&seamless querying&依赖 -Designer Developers QA Goals Allow&oodt filemanager linear scalaing&依赖 -they&local and remote querying&依赖 -" Distributed " aspect&OODT&AGGREGATION -they&file manager&依赖 -local and remote querying&file manager&AGGREGATION -I&UK&依赖 -it&more sense&依赖 -I&UK&依赖 -it&data transfer and performance perspective&依赖 -I&OODT installation&依赖 -I&UK&依赖 -I&OODT installation&依赖 -FM&information& -I&UK&依赖 -I&OODT installation&依赖 -I&OODT installation&依赖 -my&sites& -opsui query execution location a user&file manager client&依赖 -they&opsui query execution location a user&依赖 -they&file manager client&依赖 -they&service&依赖 -it&local file manager&依赖 -opsui query execution location a user&file manager client&依赖 -File Manager&remote node&依赖 -Schema&query&依赖 -Schema&query&依赖 -you&filemgr client&依赖 -simplistic implementation&definition&依赖 -definition&multiple local and remote file manager&AGGREGATION -simplistic implementation&service&依赖 -simplistic implementation&multiple local and remote file manager&依赖 -grouping&available file manager&AGGREGATION -OPSUI configuration&feature&实现 -summary&available file manager&依赖 -summary&grouping&依赖 -PCS platform&you&依赖 -PCS platform&opsui instance&依赖 -PCS platform&1 file manager&依赖 -user&summary page&依赖 -Registration&content&依赖 -lookup&available filemanager&AGGREGATION -Registration&content&依赖 -Registration&display&依赖 -Registration&content&依赖 -Registration&multiple file manager&AGGREGATION -Registration&display&依赖 -Registration&content&依赖 -display&content&AGGREGATION -Registration&display&依赖 -Registration&display&依赖 -Phase 2 Phase 2&enhancement&AGGREGATION -Phase 2 Phase 2&further distributed capability&依赖 -graceful handling&node&AGGREGATION -Phase 2 Phase 2&further distributed capability&依赖 -scalability&platform and resiliency&AGGREGATION -type deployment&distributed location&依赖 -I&OODT&依赖 -type deployment&scalability&依赖 -type deployment&platform and resiliency&依赖 diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/File Manager Scale Out Planning - OODT - Apache Software Foundation.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/File Manager Scale Out Planning - OODT - Apache Software Foundation.txt deleted file mode 100644 index 4d392f092f1e1d48e7df22228826768f1c7e62aa..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/doc/Apache OODT File Manager/File Manager Scale Out Planning - OODT - Apache Software Foundation.txt +++ /dev/null @@ -1,46 +0,0 @@ - -转至元数据结尾 -由 Tom Barber创建, 最终由 Lewis John McGibbney修改于 三月 23, 2016转至元数据起始 -Goals -Background and strategic fit -Assumptions -Requirements -User interaction and design -Phase 1 -Phase 2 -Questions -Not Doing -Target release 0.13 -Epic -Document status DRAFT -Document owner -Tom Barber - -Designer -Developers -QA -Goals -Allow for OODT filemanager linear scalaing, distribution and seamless querying of multiple filemanagers on separate networks. -Background and strategic fit -To truely fit the "Distributed" aspect of OODT, the file manager component should allow for multiple file managers to be started and they allow for both local and remote querying of the file managers seamlessly. For example, I have an OODT installation in the UK and an OODT installation on a separate site in Australia, it makes more sense from a data transfer and performance perspective to allow those 2 FM's to operate independantly of each other, but allow for querying of both repositories as if they were one, so I could choose to retieve information from the Australia FM. This also allows users to say "give me all Excel files from all my sites", without having to point fm-client at all the different instances. - -Assumptions -Requirements -Mutliple file manager configuration and monitoring via PCS Developers should be able to define multiple file managers in the PCS so they are exposed to services like OPSUI -Query execution location A user using a file manager client, should be able to tell the file manager client it only wants to search the local file manager, or search a specific remote, or search every available file manager for data. -Coping with broken communications The File Manager should be able to cope with remote nodes going offline or becoming unavailable and fail gracefully. -Schema alignment/matching A user wishes to execute a query without knowing the underlying data model, on more than one file manager server. The query should be executed regardless and the relevant content returned. Must Have -User interaction and design -If you use filemgr client then a very simplistic implementation would be to extend the service to allow definitions of multiple local and remote file managers and allow the user to execute a query over each file manager and concat the result. This would be very quick to implement but doesn't scale well or support other services. - -Phase 1 -Currently OPSUI and the PCS platform only allows you to define 1 file manager per opsui instance, but to implement this feature successfully the OPSUI configuration needs to allow for multiple file managers, and when a user looks at the summary page the summary should show a grouping of all the available file managers, whilst also allowing the ability to filter by file manager. - -In summary the following changes will be made: - -Registration of multiple file managers within PCS/OPSUI -Changes made to the OPSUI monitoring pages to reflect multiple filemanagers -Changes made to the Product listing pages to allow display of content from multiple filemanagers and filter by file manager -Additional configuration within filemgr client to allow for lookup of available filemanagers from PCS and query them. -Phase 2 -Phase 2 of the enhancement would involve adding further distributed capabilities to the filemanager(and possibly PCS platform as a whole), by adding in an optional zookeeper configuration that would allow for nodes self registering, graceful handling of nodes disappearing and also leader election and so forth. I feel enhancing OODT with industry standard distributed configuration management that is already widely used in "Big Data" type deployments will help with scalability of the platform and resiliency over distributed locations. \ No newline at end of file diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/File Manager Scale Out Planning - OODT - Apache Software Foundation.txt.xml.xls b/src/main/resources/sdtocode/doc/Apache OODT File Manager/File Manager Scale Out Planning - OODT - Apache Software Foundation.txt.xml.xls deleted file mode 100644 index 8b1dec8525dd265cb9474b47539802cd949cca9e..0000000000000000000000000000000000000000 Binary files a/src/main/resources/sdtocode/doc/Apache OODT File Manager/File Manager Scale Out Planning - OODT - Apache Software Foundation.txt.xml.xls and /dev/null differ diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Interface Ingester-relation.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/Interface Ingester-relation.txt deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Interface Ingester.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/Interface Ingester.txt deleted file mode 100644 index 64874cd0ab7710c92ba8961cff86e020932c7839..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Interface Ingester.txt +++ /dev/null @@ -1,26 +0,0 @@ -Catalog and Archive File Management Component 0.12 API -Packages -Package Description -org.apache.oodt.cas.filemgr.catalog -org.apache.oodt.cas.filemgr.catalog.solr -org.apache.oodt.cas.filemgr.cli.action -org.apache.oodt.cas.filemgr.datatransfer -org.apache.oodt.cas.filemgr.exceptions -org.apache.oodt.cas.filemgr.ingest -org.apache.oodt.cas.filemgr.metadata -org.apache.oodt.cas.filemgr.metadata.extractors -org.apache.oodt.cas.filemgr.metadata.extractors.examples -org.apache.oodt.cas.filemgr.repository -org.apache.oodt.cas.filemgr.structs -org.apache.oodt.cas.filemgr.structs.exceptions -org.apache.oodt.cas.filemgr.structs.query -org.apache.oodt.cas.filemgr.structs.query.conv -org.apache.oodt.cas.filemgr.structs.query.filter -org.apache.oodt.cas.filemgr.structs.type -org.apache.oodt.cas.filemgr.structs.type.examples -org.apache.oodt.cas.filemgr.system -org.apache.oodt.cas.filemgr.system.auth -org.apache.oodt.cas.filemgr.tools -org.apache.oodt.cas.filemgr.util -org.apache.oodt.cas.filemgr.validation -org.apache.oodt.cas.filemgr.versioning \ No newline at end of file diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Interface Ingester.txt.xml.xls b/src/main/resources/sdtocode/doc/Apache OODT File Manager/Interface Ingester.txt.xml.xls deleted file mode 100644 index 66d262bc44c34e8f4af29a28dbc6872c008f64c3..0000000000000000000000000000000000000000 Binary files a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Interface Ingester.txt.xml.xls and /dev/null differ diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Mahasen Distributed Storage Resource Broker-relation.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/Mahasen Distributed Storage Resource Broker-relation.txt deleted file mode 100644 index 94de539222ff6a6e340f34e813b419d0bbc43832..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Mahasen Distributed Storage Resource Broker-relation.txt +++ /dev/null @@ -1,664 +0,0 @@ -L’archiveouvertepluridisciplinaire HAL&deslaboratoire public oupriv�s&依赖 -L’archiveouvertepluridisciplinaire HAL&deslaboratoire public oupriv�s&依赖 -L’archiveouvertepluridisciplinaire HAL&deslaboratoire public oupriv�s&依赖 -Modern day system&avalanche&依赖 -avalanche&datum&AGGREGATION -Modern day system&datum&依赖 -datum&many forms and shape&依赖 -weather sensor&large amount&依赖 -weather sensor&datum&依赖 -weather sensor&year&依赖 -weather sensor&datum&依赖 -weather sensor&large amount&依赖 -weather sensor&large amount&依赖 -weather sensor&year&依赖 -weather sensor&datum&依赖 -weather sensor&year&依赖 -weather sensor&year&依赖 -large amount&datum&AGGREGATION -weather sensor&large amount&依赖 -weather sensor&datum&依赖 -similar datum&scalable , efficient , reliable and very large storage&依赖 -similar datum&scalable , efficient , reliable and very large storage&依赖 -similar datum&efficient metada&依赖 -paper&built&依赖 -paper&high volume datum intensive application&依赖 -paper&high volume datum intensive application&依赖 -paper&built&依赖 -paper&present mahasen&依赖 -paper&built&依赖 -top&peer-to-peer layer&AGGREGATION -paper&present mahasen&依赖 -paper&high volume datum intensive application&依赖 -paper&present mahasen&依赖 -Mahasen&weather datum&依赖 -top&distribute hash table ( dht ) 1 introduction currently united states&AGGREGATION -sizable amount&datum&AGGREGATION -sensor&datum&依赖 -sensor&sizable amount&依赖 -Processing&understanding&依赖 -its&limits& -our&understanding& -Processing&large-scale data process&依赖 -Processing&large-scale data process&依赖 -Processing&understanding&依赖 -prominent one&many challenge datum&依赖 -scientist and researcher&specific type&依赖 -specific type&datum&AGGREGATION -scientist and researcher&datum&依赖 -prominent one&datum&依赖 -scientist&Automated Weather data item&依赖 -scientist&8am-12pm&依赖 -scientist&example&依赖 -we&example&依赖 -we&meteorology&依赖 -Sky server [ 1 ]&instance&依赖 -best example&use case&依赖 -best example&large data generation&依赖 -one&best example&AGGREGATION -use case&large data generation&AGGREGATION -Sky server [ 1 ]&best example&依赖 -40 terabyte&datum&AGGREGATION -project&datum&依赖 -project&40 terabyte&依赖 -its&collection& -similarly many science&large amount&依赖 -similarly many science&large amount&依赖 -similarly many science&datum&依赖 -similarly many science&large amount&依赖 -similarly many science&datum&依赖 -similarly many science&datum&依赖 -system&file&依赖 -system&datum&依赖 -[ 2 ] [ 3 ] and storage solution&4 ] [ 5 ]&依赖 -we&related work section&依赖 -most¢ralized architecture&依赖 -most&metadata catalog implementation&AGGREGATION -most¢ralized architecture&实现 -vendor&mechanism& -centralized metadata catalog&’s mechanism&依赖 -Nirvana Storage [ 7 ]¢ralized metadata catalog&依赖 -centralized metadata catalog&scalability&依赖 -centralized metadata catalog&Oracle Real Application cluster&依赖 -Nirvana Storage [ 7 ]&example&依赖 -store&which&依赖 -store&hierarchical rich metada&依赖 -paper&scalable metadata catalog and storage server&依赖 -top&P2P technology&AGGREGATION -paper&Mahasen&依赖 -storage server&server&GENERALIZATION -paper&built&依赖 -a datum grid management system ( dgms )&distributed datum&依赖 -a datum grid management system ( dgms )&large volume&依赖 -large volume&distributed datum&AGGREGATION -It&high volume data intensive application&依赖 -architecture&Mahasen&AGGREGATION -its&attributes& -it&metadata structure&依赖 -It&storage server&依赖 -dual purpose&metadata catalog&AGGREGATION -network&dual purpose&依赖 -network&storage server&AGGREGATION -network&metadata catalog&依赖 -Mahasen&huge data storage problem and fault tolerance&依赖 -Mahasen&data intensive compute&依赖 -single point&failure&AGGREGATION -Metadata management&search file&依赖 -Metadata management&capability&依赖 -attribute&stored resource&AGGREGATION -Mahasen&metadata catalog&依赖 -metadata layer&fault tolerance&依赖 -metadata layer&replica&依赖 -metadata layer&metada&依赖 -replica&metada&AGGREGATION -rest&paper&AGGREGATION -rest&follow&依赖 -next section&related work&依赖 -next section&Metadata catalog&依赖 -section&Mahasen architecture&依赖 -Mahasen architecture&architecture&GENERALIZATION -performance evaluation&Mahasen&AGGREGATION -next section&Mahasen&依赖 -next section&performance evaluation&依赖 -discussion section§ion&GENERALIZATION -discussion section&limitation&依赖 -middleware system&large heterogeneous data resource&依赖 -centralized metadata repository&two type&依赖 -two type&records – system&AGGREGATION -centralized metadata repository&records – system&依赖 -vendor&mechanisms& -Scalability&MCAT&AGGREGATION -stored resource&Physical resource&依赖 -Replication&availability and recoverability&依赖 -Replication&resource&依赖 -Replication&resource&依赖 -Replication&availability and recoverability&依赖 -Replication&availability and recoverability&依赖 -Replication&resource&AGGREGATION -Replication&availability and recoverability&依赖 -Replication&resource&依赖 -Replication&availability and recoverability&依赖 -Replication&availability and recoverability&依赖 -Replication&resource&依赖 -Replication&resource&依赖 -Replication&resource&依赖 -availability and recoverability&resource&AGGREGATION -data transfer process&user&依赖 -datum&two way&依赖 -datum&Registration and Ingestion&依赖 -Registration&datum&依赖 -Ingestion®istration&依赖 -file resource&resource&GENERALIZATION -srb then file&MCAT&依赖 -a datum object&file resource&依赖 -2.2 Apache OODT OODT [ 10 ]&metada&依赖 -functionality&distribute datum , object and database&依赖 -functionality&distribute datum , object and database&依赖 -product service and profile service&data and metada&依赖 -OODT&file-based storage&依赖 -OODT&distributed manner&依赖 -OODT&data product&依赖 -They&three category&依赖 -They&on-line , near-line or off-line storage&依赖 -They&storage&依赖 -OODT&request&依赖 -it&profile query&依赖 -it&product server&依赖 -OODT&file&依赖 -product server&server&GENERALIZATION -form&URI&AGGREGATION -target product server address&form&AGGREGATION -response&URI&依赖 -it&datum&依赖 -it&product server&依赖 -it&product server&依赖 -oodt issue&product query&依赖 -oodt issue&product query&依赖 -it&datum&依赖 -oodt issue&product query&依赖 -it&server&依赖 -type&server&AGGREGATION -multiple&type&AGGREGATION -OODT&profile server&依赖 -profile server&server&GENERALIZATION -it&multiple&依赖 -it&type&依赖 -it&REST-style architectural pattern&依赖 -OODT&client server architecture&依赖 -it&search&依赖 -profile&retrieval&依赖 -profile&profile&AGGREGATION -implementation&javax.sql.datasource interface&AGGREGATION -file management component&resource files and metada&依赖 -file management component&delete&实现 -file management component&resource files and metada&依赖 -file management component&resource files and metada&实现 -file management component&resource files and metada&实现 -file management component&resource files and metada&实现 -file management component&delete&实现 -file management component&resource files and metada&实现 -file management component&delete&实现 -file management component&delete&实现 -file management component&delete&依赖 -file management component&Catalog and Archive Service&AGGREGATION -file management component&delete&依赖 -delete&resource files and metada&AGGREGATION -file system&system&GENERALIZATION -javax.sql.datasource interface&resource&依赖 -javax.sql.datasource interface&user&依赖 -WSO2 Registry&metadata management feature&依赖 -user&resource&依赖 -their&properties& -user&custom property&依赖 -user&custom property&依赖 -user&resource&依赖 -WSO2 Registry&resource&依赖 -WSO2 registry&Relational Database system&依赖 -it&database feature&依赖 -datum , metada&them&依赖 -it¢ralized architecture&依赖 -Mahasen&architecture&依赖 -Mahasen&distributed architecture&依赖 -distributed architecture&architecture&GENERALIZATION -replication&resource&AGGREGATION -two type&search&AGGREGATION -One&resource&实现 -One&metadata etc.&实现 -One&name&实现 -their&name& -second one&resource&依赖 -content&resource&AGGREGATION -second one&content&依赖 -second search&resource&依赖 -second search&textual content&依赖 -textual content&content&GENERALIZATION -it&property&依赖 -property&name value pair&AGGREGATION -it&name value pair&依赖 -it&tag&依赖 -it&additional metada&依赖 -your&way& -other way&own way&依赖 -other way&own way&依赖 -major limitation&memory&依赖 -amount&memory&AGGREGATION -major limitation&memory&依赖 -available memory&megabyte&依赖 -it&java heap memory&依赖 -few hundred&megabyte&AGGREGATION -hdf&master slave architecture&依赖 -master and number&DataNodes&AGGREGATION -namespace&file system&AGGREGATION -user&directory and store file&依赖 -Hadoop&hierarchical file organization&依赖 -sequence&block&AGGREGATION -block&file system&依赖 -file system&DataNodes&AGGREGATION -It&file&依赖 -block&DataNodes&依赖 -It&chunk&依赖 -default size&64MB&AGGREGATION -block&fault tolerance&依赖 -replication factor&datum&AGGREGATION -application&streaming access&依赖 -application&data set&依赖 -their&sets& -Data node&read request&依赖 -requirement&distributed file system&AGGREGATION -file&system specific metada&依赖 -physical location&system specific metada&AGGREGATION -file&using&依赖 -it&distributed file system&依赖 -it&requirement&依赖 -hadoop address&single writer multiple readers ’ model&依赖 -one&datum&依赖 -datum&file&依赖 -file&user&依赖 -particular resource&hdf&依赖 -resource&/ trash directory&依赖 -it&trash&依赖 -resource&possibility&依赖 -its&scalability& -Mahasen&differentiation& -it&large file transfer&依赖 -scalability&system&依赖 -it&an atom base resource transfer&依赖 -It&Nirvana Storage&依赖 -they&user-defined metada&依赖 -Mahasen&layer& -hdf&single name node&依赖 -single name node&active passive failover configuration&依赖 -single name node&fault tolerant&依赖 -Mahasen&several storage node&依赖 -node&metada&依赖 -node&a registry&依赖 -node&physical file part&依赖 -node&system&依赖 -node&dht ( freepastry ) route protocol&依赖 -node&communicate&依赖 -Mahasen High Level Architecture Mahasen&WSO2 registry&依赖 -Mahasen&peer&依赖 -peer network&network&GENERALIZATION -stores datum&peer network&依赖 -stores datum&peer network&依赖 -Mahasen&distributed metadata layer&依赖 -stores datum&peer network&依赖 -two main type&metada&AGGREGATION -mahasen store&metada&依赖 -mahasen store&two main type&依赖 -System defined metada&server side resource handling&依赖 -store node ip&file&AGGREGATION -example&system-defined metada&AGGREGATION -User&tags and property ( name&依赖 -User&value pair&依赖 -User&( name&依赖 -User&tags and property ( name&依赖 -User&value pair&依赖 -User&( name&依赖 -Metadata Object Structure&Mahasen&AGGREGATION -Mahasen node&node&GENERALIZATION -Metadata Object Structure&node&依赖 -master node&node&GENERALIZATION -selected set&neighborhood node&AGGREGATION -neighborhood node&master node&AGGREGATION -split part&selected set&依赖 -split part&master node&依赖 -split part&neighborhood node&依赖 -node&master node&依赖 -split part¶llel transfer&依赖 -metadata object&Free pastry&实现 -metadata object&PAST storage implementation&依赖 -metadata object&replica&依赖 -PAST storage implementation&Free pastry&AGGREGATION -node&storage& -node&file part&依赖 -their&parts& -node&worker node&依赖 -worker node&file part&依赖 -them&system&依赖 -stored location&file part&AGGREGATION -capability&concurrent access&AGGREGATION -worker node&stored location&依赖 -worker node&node&GENERALIZATION -lock manager&DHT&AGGREGATION -worker node&metadata object&依赖 -User&Mahasen node&依赖 -node&resource ID&依赖 -node&request and&依赖 -User&file&依赖 -location&metadata object&依赖 -it&location&依赖 -location&Mahasen node&AGGREGATION -location&file part&依赖 -it&Mahasen node&依赖 -file&user&依赖 -Deletion&heterogeneous storage system&依赖 -Deletion&single command&依赖 -node&file&依赖 -node&part&依赖 -part&file&AGGREGATION -Mahasen&node&依赖 -update request&request&GENERALIZATION -node&DHT&依赖 -metadata object&replica&依赖 -node&metadata object&依赖 -user&user-defined metada&依赖 -node&update request&依赖 -node&file&依赖 -Mahasen&complete decentralized metadata system&依赖 -complete decentralized metadata system&scalable and efficient manner&依赖 -complete decentralized metadata system&metadata management&依赖 -replica&both actual file and metada object&AGGREGATION -Mahasen&both actual file and metada object&依赖 -Mahasen&replica&依赖 -main purpose&replica&AGGREGATION -high availability&metada&AGGREGATION -We&high availability&依赖 -We&metada&依赖 -pastry&DHT& -amount&datum&AGGREGATION -3.2 Mahasen&complexity&依赖 -3.2 Mahasen&search increase&依赖 -complexity&search increase&AGGREGATION -3.2 Mahasen&Mahasen&GENERALIZATION -Mahasen&that&依赖 -Mahasen&a distribute datum structure&依赖 -DHT&different search option&依赖 -DHT&performance&依赖 -Mahasen&using&依赖 -performance&different search option&AGGREGATION -we&pointing&依赖 -resource&tag or property&依赖 -we&index&依赖 -resource&metada&依赖 -replica&it&AGGREGATION -a treemap [ 16 ]&a treemap [ 16 ]&依赖 -DHT&replica&依赖 -DHT&it&依赖 -property tree&DHT&依赖 -mahasen extract&requested search&依赖 -mahasen extract&requested search&依赖 -search request&request&GENERALIZATION -execution&relevant search method&AGGREGATION -user&search request&依赖 -resource id&file&AGGREGATION -resource id&relevant property tree&依赖 -user&search request&依赖 -user&Mahasen node&依赖 -Mahasen&property name&依赖 -property name&name&GENERALIZATION -node&search request&依赖 -Mahasen&given&依赖 -it&resource id&依赖 -it&property&依赖 -current node&index&依赖 -current node&property&依赖 -node handle&node&AGGREGATION -node&specific property tree&依赖 -node&master node&依赖 -node handle&relevant resource id&依赖 -their&storage& -node handle&memory storage&依赖 -memory storage&storage&GENERALIZATION -node handle&property tree&依赖 -we&sub map&依赖 -we&sub map&依赖 -set&resource id&AGGREGATION -resource id&given property value&依赖 -resource id&file&依赖 -operation&resource id&依赖 -operation&resource id&依赖 -operation&resource id&依赖 -operation&given search query&依赖 -operation&given search query&依赖 -operation&high cost&依赖 -operation&given search query&依赖 -operation&high cost&依赖 -operation&high cost&依赖 -Complete Data Structure&property base search&依赖 -Complete Data Structure&property base search&依赖 -set&different property and tag&AGGREGATION -Mahasen Search&continuation model support&依赖 -Mahasen Search&FreePastry&依赖 -Mahasen Search&continuation model support&依赖 -Mahasen Search&FreePastry&依赖 -application&request&依赖 -node handle&request result&依赖 -application&node handle&依赖 -application&first result&依赖 -network&storage node and user&AGGREGATION -Mahasen&storage node and user&依赖 -Mahasen Client&HTTP method&依赖 -client&connection&依赖 -one&node&AGGREGATION -client&node&依赖 -client&network&依赖 -client&connection&依赖 -File content&entity&依赖 -File content&HTTP POST method&依赖 -end&file stream&依赖 -file&set&依赖 -file&predefined replication factor&依赖 -file&predefined chunk&依赖 -set&predefined chunk&AGGREGATION -reliability and performance&system&AGGREGATION -critical part&system&依赖 -critical part&reliability and performance&依赖 -placement&replica&AGGREGATION -current policy&replicated file&依赖 -current policy&set&依赖 -current policy&leaf node&依赖 -current policy&replicated file&依赖 -current policy&set&依赖 -current policy&leaf node&依赖 -current policy&Mahasen&AGGREGATION -distance&node&AGGREGATION -selection&node&AGGREGATION -client&status&依赖 -client&file transfer&依赖 -initial node&other node&依赖 -initial node&file&依赖 -status&file transfer&AGGREGATION -number©&AGGREGATION -replication factor&file&AGGREGATION -block&fault tolerance&依赖 -it&block&依赖 -it&fixed size&依赖 -block&network&依赖 -block&fixed size&AGGREGATION -system&node&依赖 -retrieval&file&AGGREGATION -user&default client&依赖 -user&external client&依赖 -default client&client&GENERALIZATION -client&upload , download , delete and search operation&依赖 -result&follow figure&依赖 -result&test&AGGREGATION -result&follow figure&依赖 -500MB size file&client&依赖 -500MB size file&upload test&依赖 -number&client increase&AGGREGATION -network congestion and background process&data replication&AGGREGATION -number&18 or 24&依赖 -number&node&AGGREGATION -number&18 or 24&依赖 -client&upload&依赖 -node&other node&依赖 -node&replica management task&依赖 -node&p2p ring&依赖 -node&replica management task&依赖 -node&other node&依赖 -node&p2p ring&依赖 -client&which&依赖 -increase&number&AGGREGATION -download file&Mahasen client&依赖 -number&client&AGGREGATION -Mahasen client&client&GENERALIZATION -single node setup&significant growth&依赖 -it&other node&依赖 -it&setup&依赖 -it&file transfer&依赖 -other available node&download time&依赖 -multiple node&system&依赖 -you&file part&依赖 -you&other available node&依赖 -Mahasen&Delete&依赖 -it&3 operation&依赖 -it&metada&依赖 -its&files& -single node&lowest possible time&依赖 -it&client&依赖 -it&aggregate result&依赖 -it&it&依赖 -we&same node&依赖 -we&same node&依赖 -search operation&operation&GENERALIZATION -we&search operation&依赖 -we&search operation&依赖 -its&architecture& -Mahasen&overall system scalable and fault tolerant&依赖 -Mahasen&metadata catalog&依赖 -Mahasen&system&依赖 -Mahasen&replication&依赖 -Mahasen&metada&依赖 -Mahasen&metada&依赖 -metadata catalog¢ralized architecture&依赖 -replica&both metadata object and property tree&AGGREGATION -Mahasen&both metadata object and property tree&依赖 -easy access&them&AGGREGATION -DHT&FreePastry&AGGREGATION -replica&actual file&依赖 -replica&metadata objects and property tree object&AGGREGATION -replica&much as keeping replica&依赖 -replica&actual file&依赖 -much as keeping replica&actual file&AGGREGATION -replica&much as keeping replica&依赖 -Mahasen&Mahasen operation&依赖 -Mahasen&many&依赖 -many&Mahasen operation&AGGREGATION -Mahasen&many&依赖 -Mahasen&correct functioning&依赖 -Mahasen&correct functioning&依赖 -correct functioning&many&AGGREGATION -Mahasen&correct functioning&依赖 -Mahasen&Mahasen operation&依赖 -Mahasen&Mahasen operation&依赖 -Mahasen&many&依赖 -important contribution&DHT&依赖 -top&DHT&AGGREGATION -important contribution&DHT&依赖 -important contribution&distributed indexing structure&依赖 -important contribution&distributed indexing structure&依赖 -important contribution&top&依赖 -important contribution&top&依赖 -important contribution&Mahasen&AGGREGATION -Mahasen&range based query&依赖 -we&earlier effort&依赖 -we&such index structure&依赖 -data structure&range based query&依赖 -we&search&依赖 -data structure&DHT&依赖 -Skip Tree Graph [ 18 ]&best candidate&依赖 -we&data structure&依赖 -one&best candidate&AGGREGATION -we&different property and datum structure&依赖 -number&entry&AGGREGATION -number&property&AGGREGATION -we&less complex solution&依赖 -we&expensive&依赖 -term&resource&AGGREGATION -it&available raw metada&依赖 -it&search operation&依赖 -large number&node&AGGREGATION -Mahasen&DHT and TreeMap&依赖 -Mahasen&combined data structure&依赖 -Mahasen&metadata object&依赖 -it&node&依赖 -network&them&依赖 -it&network&依赖 -availability&metadata object&AGGREGATION -replication factor&them&AGGREGATION -we&which&依赖 -Current Mahasen design&several limitation&依赖 -we&future work&依赖 -mahasen store&one Mahasen node&依赖 -mahasen store&property index&依赖 -memory&node&AGGREGATION -mahasen store&property index&依赖 -mahasen store&one Mahasen node&依赖 -NoSQL storage&similar assumption&依赖 -NoSQL storage&similar assumption&依赖 -We&maximum size&依赖 -We&part&依赖 -maximum size&part&AGGREGATION -one potential solution&data set&依赖 -one potential solution&size&依赖 -challenge&Mahasen&依赖 -size&data set&AGGREGATION -group&open source registry&AGGREGATION -Mahasen project&scalable storage solution&依赖 -Mahasen project&project&GENERALIZATION -user&group&依赖 -user&node&依赖 -node&group&AGGREGATION -node ( registry&storage overlay&依赖 -top&Pastry DHT algorithm&AGGREGATION -node ( registry&PAST&依赖 -property-based search&data item&AGGREGATION -Mahasen&distributed indexing structure&依赖 -Mahasen&top&依赖 -Mahasen&DHT&依赖 -user&Web Service API&依赖 -batch processing&file uploading task&AGGREGATION -SIGMOD ’00 Proceedings&2000 ACM SIGMOD international conference 2000 ) 2&AGGREGATION -2000 ACM SIGMOD international conference 2000 ) 2&datum&AGGREGATION -Management&datum&AGGREGATION -Amazon&Store& -Home and storage resource broker ( srb � )&storage resource broker ( srb � )&AGGREGATION -germany ( 2001 ) 16&germany ( 2001 ) 16&依赖 -germany ( 2001 ) 16&germany ( 2001 ) 16&依赖 -germany ( 2001 ) 16&HotOS VIII&依赖 -germany ( 2001 ) 16&germany ( 2001 ) 16&依赖 -germany ( 2001 ) 16&HotOS VIII&依赖 -germany ( 2001 ) 16&germany ( 2001 ) 16&依赖 -germany ( 2001 ) 16&Schoss Elmau&依赖 -germany ( 2001 ) 16&germany ( 2001 ) 16&依赖 -germany ( 2001 ) 16&HotOS VIII&依赖 -germany ( 2001 ) 16&HotOS VIII&依赖 -germany ( 2001 ) 16&germany ( 2001 ) 16&依赖 -germany ( 2001 ) 16&Schoss Elmau&依赖 -germany ( 2001 ) 16&Schoss Elmau&依赖 -germany ( 2001 ) 16&germany ( 2001 ) 16&依赖 -germany ( 2001 ) 16&Schoss Elmau&依赖 -germany ( 2001 ) 16&HotOS VIII&依赖 -germany ( 2001 ) 16&Schoss Elmau&依赖 -germany ( 2001 ) 16&HotOS VIII&依赖 -germany ( 2001 ) 16&Schoss Elmau&依赖 -germany ( 2001 ) 16&germany ( 2001 ) 16&依赖 -germany ( 2001 ) 16&HotOS VIII&依赖 -germany ( 2001 ) 16&HotOS VIII&依赖 -germany ( 2001 ) 16&Schoss Elmau&依赖 -germany ( 2001 ) 16&HotOS VIII&依赖 -germany ( 2001 ) 16&Schoss Elmau&依赖 -germany ( 2001 ) 16&Schoss Elmau&依赖 -germany ( 2001 ) 16&germany ( 2001 ) 16&依赖 diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Mahasen Distributed Storage Resource Broker.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/Mahasen Distributed Storage Resource Broker.txt deleted file mode 100644 index 67416259f0fde4ffff53c54c973b52b030fce5e7..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Mahasen Distributed Storage Resource Broker.txt +++ /dev/null @@ -1,141 +0,0 @@ -HAL Id:hal-01513774 -https://hal.inria.fr/hal-01513774 -Submitted on25Apr2017 -HAL is amulti-disciplinaryopenaccess -archiveforthedepositanddisseminationofsci- -entificresearchdocuments,whethertheyarepub- -lished ornot.Thedocumentsmaycomefrom -teachingandresearchinstitutionsinFranceor -abroad, orfrompublicorprivateresearchcenters. -L’archiveouvertepluridisciplinaire HAL, est -destinée audépôtetàladiffusiondedocuments -scientifiquesdeniveaurecherche,publiésounon, -émanantdesétablissementsd’enseignementetde -recherchefrançaisouétrangers,deslaboratoires -publics ouprivés. -Distributed underaCreativeCommons Attribution| 4.0InternationalLicense -Mahasen: DistributedStorageResourceBroker -K. Perera,T.Kishanthan,H.Perera,D.Madola,MalakaWalpola,Srinath -Perera -Tocitethisversion: -K. Perera,T.Kishanthan,H.Perera,D.Madola,MalakaWalpola,etal..Mahasen:Distributed -Storage ResourceBroker.10thInternationalConferenceonNetworkandParallelComputing(NPC), -Sep 2013,Guiyang,China.pp.380-392,￿10.1007/978-3-642-40820-5_32￿.￿hal-01513774￿ -Mahasen: Distributed Storage Resource Broker -K.D.A.K.S.Perera1, T Kishanthan1, H.A.S.Perera1, D.T.H.V.Madola1, Malaka Walpola1, Srinath Perera2 -1 Computer Science and Engineering Department, University Of Moratuwa, Sri Lanka. {shelanrc, kshanth2101, ashansa.perera, hirunimadola, malaka.uom}@gmail.com -2 WSO2 Lanka, No 59, Flower Road, Colombo 07, Sri Lanka -srinath@wso2.com -Abstract. Modern day systems are facing an avalanche of data, and they are being forced to handle more and more data intensive use cases. These data comes in many forms and shapes: Sensors (RFID, Near Field Communication, Weather Sensors), transaction logs, Web, social networks etc. As an example, weather sensors across the world generate a large amount of data throughout the year. Handling these and similar data require scalable, efficient, reliable and very large storages with support for efficient metadata based searching. This paper present Mahasen, a highly scalable storage for high volume data intensive applications built on top of a peer-to-peer layer. In addition to scalable storage, Mahasen also supports efficient searching, built on top of the Distributed Hash table (DHT) -1 Introduction -Currently United States collects weather data from many sources like Doppler readers deployed across the country, aircrafts, mobile towers and Balloons etc. These sensors keep generating a sizable amount of data. Processing them efficiently as needed is pushing our understanding about large-scale data processing to its limits. -Among many challenges data poses, a prominent one is storing the data and indexing them so that scientist and researchers can come and ask for specific type of data collected at a given time and in a given region. For example, a scientist may want to search for all Automated Weather data items collected in Bloomington area in June 15 between 8am-12pm. -Although we have presented meteorology as an example, there are many similar use cases. For instance, Sky server [1] is one of the best examples that illustrate the use case of large data generation. This project expects to collect 40 terabytes of data in five years. In its data collection, the photometric catalog is expected to contain about 500 distinct attributes for each of one hundred million galaxies, one hundred million stars, and one million quasars. Similarly many sciences, analytic processing organizations, data mining use cases etc., would want to store large amount of data and process them later in a selective manner. These systems often store data as files -and there have been several efforts to build large scale Metadata catalogs [2][3] and storage solutions[4][5] to support storing and searching those data items. One such example is AMGA metadata catalog [6] which was an effort to build replication and distribution mechanism for metadata catalogs. -As we discuss in the related work section, most of the metadata catalog implementations use centralized architectures and therefore have limited scalability unlike Mahasen. For example, Nirvana Storage [7] has a centralized metadata catalog which only supports scalability through vendor’s mechanism such as Oracle Real Application clusters. XML Metadata Concept catalog (XMC Cat) [8] is another centralized metadata catalog which stores hierarchical rich metadata. This paper presents Mahasen, a scalable metadata catalog and storage server built on top of a P2P technology. Further, it is built by distributing an open source centralized Data registry (WSO2 Registry). -Mahasen (Distributed Storage Resource Broker) is a Data Grid Management System (DGMS) that can manage a large volume of distributed data. It targets high volume data intensive applications. The architecture of Mahasen has been designed to present a single global logical namespace across all the stored data, and it maintains a metadata structure which can be used to search files based on its’ attributes. It is a network of storage servers that plays the dual purpose of a metadata catalog and a storage server. Mahasen will solve the huge data storage problem and fault tolerance in data intensive computing through aggregating low cost hardware while having both metadata and actual resources distributed without single point of failure. Metadata management will ensure the capability of searching files based on attributes of the stored resources. Mahasen has a metadata catalog, which is highly distributed and well scalable. The metadata layer ensures fault tolerance by keeping replicas of metadata. -The rest of the paper is organized as follows. The next section will discuss the related work in Metadata catalogs and Storage servers while comparing and contrasting them with Mahasen. The following section will discuss Mahasen architecture. The next section will present the performance evaluation of Mahasen. Finally the discussion section discusses limitations, other potential solutions and directions. -2 Related Work -2.1 Nirvana Storage -Nirvana SRB [7] is a middleware system that federates large heterogeneous data resources distributed across a network. The ability to access, manage, search and organize data across the entire SRB Federation is provided via a Global Namespace. MCAT is the centralized metadata repository which maintains two types of records – system- and user-metadata. Scalability of MCAT is achieved using database vendor’s mechanisms [9], hence limited by Relational DB scalability Limits. -Storage/Replication. The stored resources are divided as Physical resources, Logical resources and Cluster resources. Replication of resources across multiple servers ensures the availability and recoverability of resources during failovers. -Retrieve. Data stream routing is handled by SRB and TCP/IP, making the data transfer process transparent to the users.. -Search. Searching is done based on metadata attributes which are extracted and managed by the SRB. -Add/Update. Data can be added in two ways: Registration and Ingestion. Registration does not transfer any data but only creates a pointer to the data in MCAT. Ingestion is similar to registration but also transfers the data to an SRB storage resource. -Delete. If a file shadow object is used as a data object to ingest a file resource to SRB then file will be removed from MCAT but not from the physical location. -2.2 Apache OODT -OODT[10] is a middleware system for metadata that provides transparent access to the resources. It facilitates functionalities such as store, retrieve, search and analyze distributed data, objects and databases jointly. OODT provides a product service and profile service which manage data and metadata respectively. -Storage/Replication. OODT stores data product in a file-based storage in a distributed manner. They classify storage into three categories: on-line, near-line or off-line storage. -Retrieve. When OODT receives a request for retrieving a file, it issues a profile query to a product server that helps in resolving resources that could provide data. The response will include the target product server address in the form of a URI. The OODT issues a product query based on the profile query results to get the data, and it will actually retrieve data from the product server in a MIME-compliant format. -Search. OODT uses the profile server and the product server for searching the metadata and retrieve the products, and it has multiple of each type of server. OODT is based on client server architecture and it promotes REST-style architectural pattern for search and retrieve data. The profile or a subset of profile is returned for retrieval. -Add/Update. OODT provide data management including manage files and folders with the implementation of javax.sql.datasource interface. -Delete. The file management component of a Catalog and Archive Service support the delete of resource files and metadata through the implementation of javax.sql.datasource interface. -2.3 WSO2 Governance Registry -WSO2 Governance Registry [11] is a repository that allows users to store resources in a tree-structured manner, just like with a file system. However, unlike a file system, users may annotate resources using their custom properties, and also WSO2 Registry has built in metadata management features like tagging, associating resources. -However, WSO2 registry is backed by a Relational Database system, and it uses database features to store data, metadata, to manage them, and to search. Hence it has a centralized architecture. Mahasen extends that architecture to a distributed architecture. -Replication. There is no inbuilt mechanism to do the replication of resources in WSO2 registry. -Search. The WSO2 registry provides two types of searches. One is searching for a resource with their name, metadata etc., and it is implemented using underline relational database system. The second one is searching the content of resources, and implemented using Lucene [12]. The second search is only applicable to resources with textual content. -Add/Update. Adding of resources to registry can be done in two ways. First one is adding via the web interface provided by the registry. When adding a new resource, it is also possible to add additional metadata such as tags, properties of name value pairs, which later will be useful to search for that resource. The other way to add resources is by writing your own way by extending the registry API and exposing it as a web service. -The major limitation with registry, when storing resources, is the amount of memory available. Since it uses the java heap memory to buffer the resources before storing them, large files cannot be stored as the available memory is only limited to few hundred of megabytes. -2.4 Hadoop Distributed File System -Apache Hadoop Distributed File System is (HDFS)[13] is a file system designed to run on commodity hardware. HDFS has a master slave architecture that consists of a single NameNode as master and number of DataNodes. The NameNode is responsible of regulating access to files by client and managing the namespace of the file system. Generally DataNodes are deployed one per node in the cluster, and is responsible of managing storage attached to that node. -Storage / Replication. Hadoop supports hierarchical file organization where user can create directories and store files. It splits the file in to chunks with the default size of 64MB and stores them as sequence of blocks, and those blocks are stored in underlying file system of DataNodes. Those blocks are replicated for fault tolerance and the block size and the replication factor of data are configurable. -Retrieve. Applications that run on HDFS need streaming access to their data sets. Data nodes will be responsible for the read requests that issued from a user to retrieve data from the system. -Search. Hadoop Distributed File System does not provide a comprehensive search for users or applications, and it just fulfill the requirement of a distributed file system by supporting to locate the physical location of the file using the system specific metadata. -Add/Update. Writing to HDFS should be done by creating a new file and writing data to it. Hadoop addresses a single writer multiple readers’ model. Once the data is written and file is closed, one cannot remove or alter data. Data can be added to the file by reopening the file and appending new data. -Delete. When a file is deleted by a user or from an application, the particular resource is not immediately removed from HDFS. The resource will be renamed and copied in to /trash directory giving the possibility to restore as long as it remains in the trash. -Mahasen’s main differentiation from above systems comes from its scalability. It can scale significantly than Nirvana Storage that depends on relational databases to scale the system, since the Mahasen metadata layer is natively distributed using a DHT.WSO2 Registry provides the clustering as the scalability option, but it is not optimized for large file transfers and storing as it uses an ATOM based resource transfers. Furthermore, Mahasen provides users a comprehensive metadata model for managing the distributed resources they stored with user-defined metadata, unlike the HDFS, which only focuses on creating a Distributed file system. Further Mahasen's metadata layer is natively distributed and fault tolerant while HDFS has a single name node which can make fault tolerant only with an active passive failover configuration. -3 High Level Architecture -3.1 Mahasen High Level Architecture -As shown by Figure 1, Mahasen consists of several storage nodes which are connected as peers to a logical ring via FreePastry. Each node consists of a registry to store -metadata and a file system to store physical file parts. Once connected to the ring each node contributes to the metadata space as well as file storage capacity, scaling the system dynamically with new node additions. Nodes use underline DHT (FreePastry) routing protocol to communicate efficiently with each other. -Fig. 1. Mahasen High Level Architecture -Mahasen uses a WSO2 registry and the file system in each node and DHT based architecture is used to connect the nodes to a one unit. -Mahasen has a distributed metadata layer that stores data about the distributed files in Mahasen peer to peer network. The metadata catalog is used to broker the stored resources in the network and to assist the user to locate the files in Mahasen distributed environment abstracting the metadata management from the user. -Mahasen stores two main types of metadata, which are system-defined metadata and user-defined (descriptive) metadata. System defined metadata is mainly used for server side resource handling. File name, file size, stored node IPs of file are examples of the system-defined metadata. User defined metadata is used to provide users the searching capability on those metadata. User can add tags and properties (name, value pairs) to the files that are uploaded. -Fig. 2. Metadata Object Structure of Mahasen -When a file is uploaded connecting to a Mahasen node the file will be temporarily saved in that node. Then the node will act as the master node and split the file into pre-defined sized chunks and the split parts are stored in a selected set of the neighborhood nodes of master node through parallel transfer. Then the metadata object created by master node will be stored with replicas using PAST storage implementation of Free pastry. We have rewritten PAST node’s persistent storage such that the data will be stored in the WSO registry in that node. -After storing the metadata, the nodes that received file parts act as worker nodes and replicate their file parts in parallel according to the replicate request issued by the master node. Each worker node will update the metadata object with stored locations of the file parts which were replicated after replicating their file parts using the capability of concurrent access to metadata objects, and Mahasen handles them using the locking system provided by the lock manager of DHT. -User can request to download a file from any Mahasen node and the node will first generate the resource ID for the requested and retrieve the metadata object. Then it extracts the locations of Mahasen nodes that contain the file parts from the metadata object and retrieve those parts to the local machine. The parts will be merged to create the original file after retrieving all the parts and the file will be streamed to the user. -Deletion can be performed with a single command across a heterogeneous storage system. When a delete request for a file is issued, by following the same method of retrieving the file, Mahasen finds nodes that store parts of the file and deletes them. Finally the metadata object will also be deleted with replicas -When user needs to update the user-defined metadata, the node that receives the update request retrieves the metadata object for the file from the DHT, updates it, and stores it back in the DHT. -. Using this model, Mahasen has built a complete decentralized metadata system that handles metadata management in a highly scalable and efficient manner. -Mahasen keeps replicas of both actual files and metadata objects. The main purpose of keeping replicas is for fault tolerance and failover recovery. We ensure the high availability of metadata while ensuring the scalability using free pastry’s underlying DHT. -3.2 Mahasen Search -When the amount of data in the system grows, the complexity of the search increases. Mahasen builds a distributed data structure using the underlying DHT, which can improve the performance of different search options that Mahasen supports. -The resources in Mahasen are associated with metadata and for each tag or property in system, we maintain an index pointing to all resources which have that tag or property. This is implemented as a TreeMap [16] and the property trees are stored in the DHT which handles replicas of it. -Fig. 3. A Property Tree Stored in Mahasen Memory Storage -When a user sends a search request, Mahasen extracts the requested search and initiate the execution of relevant search method. Then the resource IDs of the files which match with the given input are retrieved from the relevant property tree. Extracting the relevant resource IDs are done as follow. -Users can send search requests to any Mahasen node, and when a node receives a search request, Mahasen takes the property name given by the client and generates the property tree ID for that property. If the current node has the index for the property, it receives matching resource IDs for that property and sends them to the client. If not, the node acts as a master node and gets the node handles of the nodes which are having the specific property tree and routs Mahasen search messages with the required parameters to the node handles. Then those node handles will get the relevant resource IDs from the property trees in their memory storage and send back to the master node. -The property values in the property tree are sorted, so that if the search is a range based search, we can simply take the sub map between the initial and final property values and retrieve the set of resource IDs mapped to each of the node in the sub tree. Since these resource IDs represents the files having the given property values, Mahasen can look up for the metadata objects with those resource IDs and extract the file names to present to for the user. The operation of extracting the file names for the resource IDs has a high cost than extracting the matching resource IDs for the given search query. -Complete Data Structure built for Mahasen can support property based search, range based search, tag based search and Boolean operations for the properties such as AND operation and OR operation. The advanced search provided by Mahasen is capable of providing the search based on set of different properties and tags. -Mahasen Search utilizes the continuation model support by FreePastry in results retrieving and transferring. Therefore when a search request is issued, the application sends requests to look up node handles, which contain the particular TreeMap object to request results. Then the application will collect the first result incoming and resume action from the previous execution point. -3.3 File Handling -File Transfer. Mahasen is a network of storage nodes and users will be given a client which is the Mahasen Client to access and transfer files to the network. The Mahasen Client that is built using the Apache HttpClient [17] uses HTTP methods for transferring files to the network. First the client initiates a connection with one of the node in the network. An authenticated client is capable of uploading downloading, deleting, updating or searching for the files in the network. The File content will be added as an entity to the HTTP POST method and streamed to the target address. The receiving end will read the file stream and write it to the repository. -Replica Management. To achieve fault tolerance and failover recovery, the file will be split into a set of predefined chunks and each part will be replicated and stored in different nodes according to predefined replication factor. The placement of replicas is a critical part which affects the reliability and performance of the system. The purpose of having a policy for placement of replicas is for data reliability, availability, and network bandwidth utilization. The current policy of Mahasen is to store the replicated files in leaf nodes set to the initial node. The selection of nodes in the leaf set will be calculated using cost evaluation function which focus on the distance of the node. -After successfully transferring the file to the initial node, the client will be notified about the status of the file transfer and initial node will then replicate and transfer the file to other nodes. The number of copies kept for a file is called the replication factor of that file and will be decided by the Mahasen system. -File Splitting and Parallel transfer. Mahasen storage network is designed to store large files reliably across distributed nodes. When storing the file it will be split into blocks of fixed size and these blocks will be replicated across the network for fault tolerance. The transferring of replicated file blocks will be done in parallel to other nodes in order to utilize the bandwidth and to save time. -When focusing on the retrieval of a file by using the metadata object the system will then select a node which is closest to the reader node and download the blocks to the client. Downloading of file blocks will also be done in parallel and then the blocks will be merged to create the complete file. -3.4 Mahasen API -Mahasen provides a complete API to perform CRUD operations and search. Users can develop external clients apart from the default client Mahasen provides and integrate with existing systems to perform resource management and search operations. -3 Performance Analysis -The Mahasen System Scalability was tested by running a system with M nodes and N parallel clients. Here the value for M was 1, 6, 12, 18, 24 and N was 1, 5, 10, 15, 20. Each client carried out upload, download, delete and search operations for 10 times and the average was taken. The system configuration that was used in this test are, Two machines with Intel(R) Xeon(R) CPU E5-2403 1.80GHz 4 Core machines having 24GB RAM and One machine with Intel(R) Xeon(R) CPU E5-2470 2.30GHz 8 Core machines having 63GB RAM. Following Figures (from 4 to 7) depicts the results of this test. In the upload test, 500MB size files were used by each client. . -Fig. 4. Upload test results -In the results it is observed that when the number of client increases, the upload time is also increasing. We believe that this is due to the network congestion and background processes of data replication across nodes. When the number of nodes increased to 18 or 24, a reduction in upload time were observed. This was an expected behaviour, because the node which client selects to upload, distributes replica management task for other nodes in the p2p ring. -Fig. 5. Download test results -When download files using Mahasen client, it is observed that with the increase of number of client, the single node setup has a significant growth in the download time. In the performance test, a single node was chosen to send the client request while it coordinates the file transfer from other nodes in the setup. Therefore when there are multiple nodes in the system you can download file parts from other available nodes, which reduces the download time. -Fig. 6. Delete test results -When Mahasen performs a Delete on a resource, it involves 3 operations such as deleting metadata, deleting entries from search index, and deleting the physical file. When more nodes are in the system, each node can participate in deleting its own files in parallel, making the system more scalable and efficient. -Fig 7. Search test results -Search results illustrate that Mahasen can perform well even with more nodes added to the system. Usually single node should have the lowest possible time as it does not have to search across the p2p ring. But with multiple nodes, it has to aggregate results and present it to the client. This can be observed from the figure that, when more clients are in the system, results tend to converge into a lower value due to caching as we requested search operation through the same node. -3 Discussion and future work -Mahasen provides a highly scalable metadata structure with its peer-to-peer architecture in the metadata catalog. Unlike the existing metadata catalogs that use centralized architecture, Mahasen distributes metadata across the nodes in the system with the replication making the overall system scalable and fault tolerant. -Mahasen keeps replicas of both metadata objects and property trees as well. The DHT of FreePastry is used to store these objects in the system which provides easy access of them. Keeping replicas of metadata objects and property tree objects do not cost as much as keeping replicas of actual files which are very large in size compared to metadata and property tree objects. By having these objects with replicas in the system, Mahasen has been able to ensure the correct functioning of many of the Mahasen operations even in the conditions like node failures. -An important contribution of Mahasen is developing a distributed indexing structure on top of the DHT for searching data products using different properties associated with data products. Since Mahasen needed to support range based queries, we evaluated earlier effort to build such index structures. Skip Tree Graph [18] was one of the best candidates we selected for search assisting data structure, which can efficiently support range based queries over a DHT. Since we had different properties and data structure had to grow in two dimensions, one in number of properties and the other one in number of entries for one property we were forced to create different DHTs for different properties. Therefore we needed to evaluate a much less complex -solution since maintaining different DHTs could have been very expensive in terms of resources. -When the system scales up with the large number of nodes, it will be more costly to issue a search operation on the available raw metadata stored. Therefore Mahasen developed a combined data structure with DHT and TreeMap as explained earlier. -When a Mahasen node fails, and it is detected by the existing nodes in the network, Mahasen replicates all the metadata objects and the property tree objects which were in the failed node to the existing Mahasen node reading them from other replicas. Mahasen helps in preserving the availability of metadata objects and property tree objects by maintaining the replication factor of them a constant. -Current Mahasen design has several limitations, which we plan to handle as future works. Currently Mahasen stores each property indexes in one Mahasen node and assumes that it will fit within the memory of that node. This may not be major concern for simple cases, and even NoSQL storages like Cassandra makes similar assumptions. Dividing the property tree into parts and storing them in different nodes when it is larger than a given size can solve this problem. We can predefine the maximum size of a part that will be residing in one node. -Another challenge is that search based multiple properties where at least one is a common property would force Mahasen to join large data sets, and one potential solution is to negotiate the size of data sets before start the data merging. -To summarize, Mahasen project builds a scalable storage solution by making a group of existing open source registries work as a one unit. It provides a one logical global namespace, and users may talk to any node of the group and perform any operations. -Mahasen connects nodes (registries) using PAST, a storage overlay implemented on top of Pastry DHT algorithm. Furthermore, Mahasen builds a distributed indexing structure on top of DHT to support property-based search of data items. -A user can benefit from the Web Service API provided and effectively utilize for batch processing of file uploading task through a custom client or basic client provided by Mahasen. -References -1. Alexander, S., Szalay, Peter, Z., Kunszt, Ani Thakar, Jim Gray, Don Slutz, and Robert, J., Brunner.: Designing and Mining Multi-Terabyte Astronomy Archives.: The Sloan Digital Sky Survey. In: SIGMOD ’00 Proceedings of the 2000 ACM SIGMOD international conference on Management of data (2000) -2. Chaitanya Baru, Reagan Moore, Arcot Rajasekar, Michael Wan.:The SDSC Storage Resource Broker (1998) -3. Reagan, W., Moore.: Managing Large Distributed Data Sets using the Storage Resource Broker (2010) -4. G., DeCandia, D., Hastorun, and M., Jampani.: Dynamo.: Amazon’s Highly Available Key-value Store (2010) -5. Ghemawat, S.-T., Leun, and H., Gobioff.: The Google File System. -6. B., K., Nuno Santos.: Distributed Metadata with the AMGA Metadata Catalog. -7. Nirvana Storage - Home of the Storage Resource Broker (SRB®), http://www.nirvanastorage.com/index.php?module=htmlpages&func=display&pid=1 (2011) -8. XML Metadata Concept Catalog (XMC Cat), Data to Insight Center, Indiana University Pervasive Technology Institute, http://d2i.indiana.edu/xmccat. -9. Nirvana Performance, http://www.nirvanastorage.com/index.php?module=htmlpages&func=display&pid=54. -10. ApacheTM OODT, http://oodt.apache.org/ (2011) -11. WSO2 Governance Registry - lean.enterprise.middleware - open source SOA | WSO2, http://wso2.com/products/governance-registry/ (2011) -12. Apache Lucene - Overview, http://lucene.apache.org/java/docs/index.html. -13. HDFS Architecture Guide, http://hadoop.apache.org/docs/r1.0.4/hdfs_design.html (2011) -14. Pastry - A scalable, decentralized, self-organizing and fault-tolerant substrate for peer-to-peer applications, http://www.freepastry.org/. -15. P., Druschel and A., Rowstron.: PAST: A large-scale, persistent peer-to-peer storage utility. In: HotOS VIII, Schoss Elmau, Germany (2001) -16. TreeMap (Java 2 Platform SE 5.0), http://download.oracle.com/javase/1.5.0/docs/api/java/util/TreeMap.html (2011) -17. HttpClient - HttpComponents HttpClient Overview, http://hc.apache.org/httpcomponents-client-ga/ (2011) -18. Alejandra Gonz´alez Beltr´an, Paul Sage and Peter Milligan.: Skip Tree Graph: a Distributed and Balanced Search Tree for Peer-to-Peer Networks. \ No newline at end of file diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Mahasen Distributed Storage Resource Broker.txt.xml.xls b/src/main/resources/sdtocode/doc/Apache OODT File Manager/Mahasen Distributed Storage Resource Broker.txt.xml.xls deleted file mode 100644 index c23341e922bcd6d7d8c92f421ba720227f21896e..0000000000000000000000000000000000000000 Binary files a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Mahasen Distributed Storage Resource Broker.txt.xml.xls and /dev/null differ diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/OODT Filemgr User Guide-relation.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/OODT Filemgr User Guide-relation.txt deleted file mode 100644 index 505897ac09dc5a94a30e0abb2aca2aa4713809e7..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/doc/Apache OODT File Manager/OODT Filemgr User Guide-relation.txt +++ /dev/null @@ -1,204 +0,0 @@ -最终由 nadeeshan gimhana修改于 五月 17 , 2019转至元数据起始&what&AGGREGATION -we&configuration&依赖 -brief overview&a typical user scenario a few more tool tips and tricks&依赖 -brief overview&a typical user scenario a few more tool tips and tricks&依赖 -brief overview&a typical user scenario a few more tool tips and tricks&依赖 -brief overview&FileManager The File Manager&依赖 -brief overview&FileManager The File Manager&依赖 -brief overview&FileManager The File Manager&依赖 -brief overview&filemgr-client and query-tool command&AGGREGATION -brief overview&FileManager The File Manager&依赖 -brief overview&a typical user scenario a few more tool tips and tricks&依赖 -brief overview&a typical user scenario a few more tool tips and tricks&依赖 -brief overview&FileManager The File Manager&依赖 -self&first time user&依赖 -you&page&依赖 -tutorial&mean&依赖 -tutorial&all file managers functionality&依赖 -complete overview&all file managers functionality&AGGREGATION -you&basic tool&依赖 -topic&page&依赖 -your&system& -/ filemgr/target/cas-filemgr&directory structure&依赖 -/ filemgr/target/cas-filemgr&directory structure&依赖 -/ filemgr/target/cas-filemgr&directory structure&依赖 -/ filemgr/target/cas-filemgr&directory structure&依赖 -/ filemgr/target/cas-filemgr&directory structure&依赖 -/ filemgr/target/cas-filemgr&directory structure&依赖 -you&version 0.3&依赖 -version 0.3&OODT&AGGREGATION -policy directory&directory&GENERALIZATION -you&OODT&依赖 -policy directory&this (&依赖 -policy directory&sub directory&依赖 -└ ─ ─ policy ├ ─ ─ elements.xml ├ ─ ─ product-type-element-map.xml └ ─ ─ product-types.xml&directory&依赖 -you&that&依赖 -brief description&directory&AGGREGATION -bin&configuration file&依赖 -bin&i.e. *&依赖 -bin&configuration file&依赖 -bin&i.e. *&依赖 -jar file log&contains file manager log file&依赖 -bin directory&number&依赖 -bin directory&directory&GENERALIZATION -number&executable&AGGREGATION -bin directory&executable&依赖 -You&file manager&依赖 -catalog query tool&File Manager&依赖 -/&filemgr&GENERALIZATION -you&java.net.BindException exception&依赖 -other service&port 9000&依赖 -file&filemgr process id&依赖 -you&configuration&依赖 -your&filemgr& -You&repository path&依赖 -You&product-types.xml file&依赖 -/&/&GENERALIZATION -path&repository path xml element&依赖 -your&xml& -it&filemgr.properties and product-type&依赖 -we&what&依赖 -your&catalog& -database&metada&AGGREGATION -place&metada&依赖 -place&metada&依赖 -place&database&依赖 -place&database&依赖 -your&files& -place&repository&依赖 -place&repository&依赖 -location&policy directory&AGGREGATION -your&directory& -your mime-types configuration file&file recognition&依赖 -your mime-types configuration file&file recognition&依赖 -your mime-types configuration file&file recognition&依赖 -your mime-types configuration file&your mime-types configuration file&依赖 -your mime-types configuration file&your mime-types configuration file&依赖 -your mime-types configuration file&your mime-types configuration file&依赖 -Your&mime-types& -your mime-types configuration file&file recognition&依赖 -your mime-types configuration file&your mime-types configuration file&依赖 -metada&brief note&依赖 -metada&how&依赖 -filemgr&metada&依赖 -filemgr&client side metada extraction and server side metada extraction&依赖 -filemgr&two different way&依赖 -metadata file&file&GENERALIZATION -Client side metada&filemgr&依赖 -Client side metada&xml formatted metadata file&依赖 -a file call blah.txt&metadata file&依赖 -extractor&chosen policy directory&依赖 -extractor&product-types.xml file&依赖 -you&/ usr/local/oodt / cas-filemgr/policy&依赖 -you&example configuration&依赖 -you&have&依赖 -you&policy directory&依赖 -you&policy directory&依赖 -you&/ usr/local/oodt / cas-filemgr/policy&依赖 -version 0.3 or earlier&OODT&AGGREGATION -you&policy directory&依赖 -example configuration&configuration&GENERALIZATION -you&version 0.3 or earlier&依赖 -you&cas-filemgr/policy/oodt&依赖 -product-types.xml file&file&GENERALIZATION -we&repository&依赖 -our&file& -It&critical information&依赖 -we&first file&依赖 -It&going&依赖 -we&that&依赖 -GenericFile type& key&依赖 -It&metada&依赖 -We&product type&依赖 -GenericFile type& key&依赖 -It&extractor&依赖 -more detail&Metadata Extractors&依赖 -more detail&Metadata Extractors&依赖 -we&product types and element&依赖 -reader&discussion&依赖 -discussion&best practice&AGGREGATION -reader&Everything&依赖 -you&File Manager Policy&依赖 -reader&w.r.t File Manager Policy&依赖 -reader&best practice&依赖 -command&/ usr/local/oodt / cas-filemgr/bin&依赖 -brief overview&filemgr-client and query-tool&AGGREGATION -we&filemgr-client&依赖 -number&different way&AGGREGATION -location&filemgr xml-rpc data transfer interface&AGGREGATION -INFO statement&us&依赖 -we&that&依赖 -13 different type&operation&AGGREGATION -we&-- ingestProduct operation&依赖 -you&further command line argument&依赖 -I&light&依赖 -we&ingestProduct&依赖 -your&repository& -content&repository&AGGREGATION -/ query-tool Must&a query and filemgr url&依赖 -anything&query tool&AGGREGATION -SELECT&GenericFile& -query&SELECT& -it&us&依赖 -you&look&依赖 -a typical user scenario time&simple file&实现 -a typical user scenario time&a typical user scenario time&依赖 -a typical user scenario time&a typical user scenario time&实现 -a typical user scenario time&simple file&依赖 -it&$ cd / usr/local/oodt / cas-filemgr/bin $&依赖 -it&filemgr.properties&依赖 -we&client side metada&依赖 -metadata collection&product-types.xml file&依赖 -metadata collection&specified * Extractor extractor&依赖 -metadata collection&server side&依赖 -its&file& -tmp/blah&text file&依赖 -tmp/blah&text file&依赖 -tmp/blah&ingestion&依赖 -text file&file&GENERALIZATION -hello&tmp/blah& -tmp/blah&ingestion&依赖 -blah.txt.met let&file&依赖 -we&arguments :&依赖 -you&ingested product&依赖 -your&product& -we&product location&依赖 -we&whole directory&依赖 -we&one product&依赖 -/&tmp/blah&GENERALIZATION -output&sep 16 , 2011 2:09:42 pm org.apache.oodt.cas.filemgr.system.xmlrpcfilemanagerclient &依赖 -You&your first file (&依赖 -your&file& -we&metada&依赖 -Query&/&依赖 -output&sep 16 , 2011 2:21:54 pm org.apache.oodt.cas.filemgr.system.xmlrpcfilemanager complexquery info&依赖 -output&sep 16 , 2011 2:21:54 pm org.apache.oodt.cas.filemgr.system.xmlrpcfilemanager complexquery info&依赖 -file&archive&依赖 -Query command&catalog implementation&依赖 --- sql and -- lucene&filemgr query syntax&依赖 -query&tutorial&依赖 -query&time&依赖 -query&tutorial&依赖 -query&time&依赖 -small deviation&unexpected value&依赖 -small deviation&exception&依赖 -Formatting&query&AGGREGATION -small deviation&query return&依赖 -small deviation&unexpected value&依赖 -small deviation&exception&依赖 -small deviation&query return&依赖 -single quote ('')&Count&依赖 -single quote ('')&WHERE clause&依赖 -single quote ('')&command line option&依赖 -=&Count& -single quote ('')&string value&依赖 -order&return value&AGGREGATION -you&\ outputFormat option&依赖 -verbose example&aware&依赖 -verbose example&SQL-like syntax&依赖 -CAS.ProductReceivedTime&\& -=&blah.txt& -sortBy&\& -WHERE&blah.txt& -a few more tool cameron goodale&some useful command line tool alias&依赖 -My&Catalog& diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/OODT Filemgr User Guide.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/OODT Filemgr User Guide.txt deleted file mode 100644 index d54f7ddc463251b78178bfc3b81e55ec2ccf5216..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/doc/Apache OODT File Manager/OODT Filemgr User Guide.txt +++ /dev/null @@ -1,299 +0,0 @@ -OODT Filemgr User Guide -页面… CAS User Guides -跳到banner的尾部 -回到标题开始 -转至元数据结尾 -由 Thomas Bennett创建, 最终由 Nadeeshan Gimhana修改于 五月 17, 2019转至元数据起始 -The File Manager -An Overview of What is Installed -Configuring and Running the File Manager -Whats going to happen? -Now for some configuration -What have we configured? -How metadata is collected? -A brief overview of filemgr-client and query-tool -Command: filemgr-client -Command: query-tool -A Typical User Scenario -A few more tools -Tips and Tricks for FileManager -The File Manager -This self guided tutorial is intended for first time users. - -The fact that you've found this page, I assume that you are seriously thinking of using the OODT File Manager but are eager to get something up and running. It hopefully also means that you've checked out the code and built a cas-filemgr install target (e.g. a cas-filemgr-${version}-dist.tar.gz file). - -This tutorial is by no means a complete overview of all the File Managers functionality. However, it's an attempt to get you started using the basic tools. Like learning to drive a car, the most difficult part is getting it started and on the road! - -The following topics are covered on this page: - -An Overview of What is Installed -Configuring and Running the File Manager -A Typical User Scenario - ingesting and querying -An Overview of What is Installed -Assumption - you have built or have access to a cas-filemgr install target. This also means that you've correctly configured maven and java for your system. - -Here are the commands to install the cas-filemgr target from a tarfile. You will need to fit in the "..." with the appropriate content. - -$ mkdir -p /usr/local/oodt/ -$ tar xzvf .../filemgr/target/cas-filemgr-${version}-dist.tar.gz -C /usr/local/oodt/ -$ cd /usr/local/oodt/ -$ ln -s cas-filemgr-${version}/ cas-filemgr -The decompressed tar file creates a directory structure that looks as follows: - -. -├── bin -│ ├── filemgr -│ ├── filemgr-client -│ └── query-tool -├── etc -│ ├── filemgr.properties -│ └── mime-types.xml -├── lib -│ └── *.jar -├── logs -└── policy -| ├── cmd-line-actions.xml -| ├── cmd-line-options.xml -| ├── core -| │ ├── elements.xml -| │ ├── product-type-element-map.xml -| │ └── product-types.xml -| | -| ├── trace -| | ├── elements.xml -| | ├── product-type-element-map.xml -| | └── product-types.xml -| | -| ├── geo -| | ├── elements.xml -| | ├── product-type-element-map.xml -| | └── product-types.xml -| | -| (additional policy sub directories) -└── run -Please note, if you are using version 0.3 of OODT or earlier, the policy directory will look like this (with no sub directories): - -└── policy - ├── elements.xml - ├── product-type-element-map.xml - └── product-types.xml -Here is a brief description of each directory that you see listed: - -bin : contains shell convenience scripts for launching java classes -etc : contains configuration files, i.e. *.property and *.xml files -lib : contains java resources, i.e *.jar files -logs : contains file manager log files. -policy : contains product specifications, i.e *.xml specification files -The bin directory contains a number of executables: - -filemgr : file manager (startup/shutdown) script -filemgr-client : file manager client interface script -query-tool : catalog query tool -Configuring and Running the File Manager -You're now ready to run the file manager! - -$ cd /usr/local/oodt/cas-filemgr/bin -$ ./filemgr --help -Usage: ./filemgr {start|stop|status} -$ ./filemgr start -Whats going to happen? -The filemgr should be up and running, however, some WARNING messages may appear, complaining about configuration. - -If you get a java.net.BindException exception, make sure that no other service is running on port 9000. This is the port for an RPC interface that will be used for transferring data files into a repository. - -There's also a new file in the /usr/local/oodt/run directory. The file contains the filemgr process id. This is typical for *nix service house keeping. It is done to try and avoid running multiple filemgr services. - -There's also a new log file /usr/local/oodt/cas-filemgr/logs/cas_filemgr0.log. Tailing this file can often alert to you problems. - -$ tail -f /usr/local/oodt/cas-filemgr/logs/cas_filemgr0.log - -Now for some configuration -To do anything useful with your filemgr, you will need to specify some configurations in the /usr/local/oodt/cas-filemgr/etc/filemgr.properties file. - -Here is a basic modification to the filemgr.properties file: - -filemgr.properties -org.apache.oodt.cas.filemgr.catalog.lucene.idxPath=/usr/local/oodt/cas-filemgr/catalog -org.apache.oodt.cas.filemgr.repositorymgr.dirs=file:///usr/local/oodt/cas-filemgr/policy/core -org.apache.oodt.cas.filemgr.validation.dirs=file:///usr/local/oodt/cas-filemgr/policy/core -org.apache.oodt.cas.filemgr.mime.type.repository=/usr/local/oodt/cas-filemgr/etc/mime-types.xml -You will also need to specify a repository path in the product-types.xml file. Make sure that this path exists before you change the repository path xml element. - -product-types.xml - -Restart your filemgr so that it re-reads the filemgr.properties and product-types.xml: -$ cd /usr/local/oodt/cas-filemgr/bin -$ ./filemgr restart - -What have we configured? -A place to store your catalog, i.e. the database of metadata. -A place to store your ingested files, i.e. the repository. -The location of your policy directory for product specifications. -Your mime-types configuration file for file recognition. -How metadata is collected? -Now for some brief notes about how metadata is collected. The filemgr captures metadata in two different ways - from client side metadata extraction and server side metadata extraction. - -Client side metadata is passed to the filemgr via an xml formatted metadata file. E.g. a file called blah.txt can have a metadata file called blah.txt.met. This met file can be created in many ways, even by hand! And thats exactly what we're going to do. - -Server side metadata is generated by using java classes and the extractors that will be used are configured in the product-types.xml file in the chosen policy directory. For this example configuration, you should have /usr/local/oodt/cas-filemgr/policy/oodt as the policy directory, unless you're running version 0.3 or earlier of OODT, in which case you should have /usr/local/oodt/cas-filemgr/policy as the policy directory. - -Now would be a good time to have a quick look at the product-types.xml file. It contains some critical information about what is going to happen when we ingest our first file into the repository. - -Specified in the product-types.xml file, there is a default product type called GenericFile. This is the product type that we are going to use for the first file for ingestion. - -For the GenericFile type find the key. It's specifying some metadata. We're defining the product type! - -For the GenericFile type find the key. It's specifying some extractors to use for server side metadata extraction, namely: CoreMetExtractor, MimeTypeExtractor, FinalFileLocationExtractor. For more details about metadata and extractors see Metadata Extractors. - -If you're feeling curious, check out the other xml files in the /usr/local/oodt/cas-filemgr/policy subdirectories to get a better feel for how we define product types and elements. For a discussion of best practices w.r.t File Manager Policy, the reader is referred to Everything you want to know about File Manager Policy - -A brief overview of filemgr-client and query-tool -These commands are found in /usr/local/oodt/cas-filemgr/bin. - -Command: filemgr-client -In order to trigger a file ingestion we're going to use the filemgr-client. This is by no means the most automated way to ingest data into an repository, however it's a really easy and intuitive way to trigger a file ingestion. The filemgr-client is a wrapper script, making it easier to invoke a java executable from the command line. - -$ cd /usr/local/oodt/cas-filemgr/bin -$ ./filemgr-client --help -filemgr-client --url --operation [ [params]] -operations: ---addProductType --typeName --typeDesc - --repository --versionClass ---ingestProduct --productName --productStructure - --productTypeName --metadataFile - [--clientTransfer --dataTransfer ] - --refs ... ---hasProduct --productName ---getProductTypeByName --productTypeName ---getNumProducts --productTypeName ---getFirstPage --productTypeName ---getNextPage --productTypeName --currentPageNum ---getPrevPage --productTypeName --currentPageNum ---getLastPage --productTypeName ---getCurrentTransfer ---getCurrentTransfers ---getProductPctTransferred --productId --productTypeName ---getFilePctTransferred --origRef -As you can see there's a number of different ways this command can be executed. - -The first command line argument is --url. This is the location of the filemgr xml-rpc data transfer interface. Looking at the filemgr logs (specifically cas_filemgr0.log), we see an INFO statement telling us that local data transfer is enable on http://localhost:9000. This is the url that we need to specify. - -The second command line argument is --operation and there are 13 different types of operations that are possible! For now we are going to use the --ingestProduct operation. From the help command you can see that the --ingestProduct operation requires some further command line arguments to be specified. - -However, before we take a look at the --operation --ingestProduct, I would first like to shed a bit more light on the query-tool command. - -Command: query-tool -This is a very useful wrapper script to query the content of your repository. - -$ cd /usr/local/oodt/cas-filemgr/bin -$ ./query-tool -Must specify a query and filemgr url! -Usage: QueryTool [options] -options: ---url - Lucene like query options: - --lucene - -query - SQL like query options: - --sql - -query - -sortBy - -outputFormat -We see that we need to set some command line arguments to get anything useful out of the query tool. Try the next command: - -$ ./query-tool --url http://localhost:9000 --sql -query 'SELECT * FROM GenericFile' - -This should throw an exception, telling us it failed to perform a query. This is because there is no catalog yet (and therefore the GenericFile information does not exist). In fact if you have a look there is no catalog directory: - -$ ls /usr/local/oodt/cas-filemgr/catalog -ls: /usr/local/oodt/cas-filemgr/catalog: No such file or directory - -A Typical User Scenario -Time to ingest a very, very simple file. If you have not already, restart your filemgr so that it re-reads the filemgr.properties: -$ cd /usr/local/oodt/cas-filemgr/bin -$ ./filemgr restart - -For this simple ingestion we are not going to include any client side metadata, all the metadata collection will happen on the server side using the specified *Extractor extractors in the product-types.xml file. - -Create a text file and its metadata file for ingestion: -$ echo 'hello' > /tmp/blah.txt -$ touch /tmp/blah.txt.met - -Add the following xml to the /tmp/blah.txt.met file: - -blah.txt.met - - -Lets ingest the file! For --operation --ingestProduct we need to specify the following arguments: - ---productName : The name you want for your ingested product ---productStructure : Flat file or directory (i.e. hierarchical). Yes... we can ingest whole directories as one product ---productTypeName : A product type (as per product-types.xml) ---metadataFile : The client side metadata file ---refs : The product location -There's also an optional argument --clientTransfer, however, we're going to leave this and use the default local transfer. -[--clientTransfer --dataTransfer ] - -Here is the complete command: -$ ./filemgr-client --url http://localhost:9000 --operation --ingestProduct --productName blah.txt --productStructure Flat --productTypeName GenericFile --metadataFile file:///tmp/blah.txt.met --refs file:///tmp/blah.txt - -The output should look like: -Sep 16, 2011 2:09:42 PM org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient -... -... -ingestProduct: Result: c2fbf4b9-e05c-11e0-9022-77a707615e7f - -You've just archived your first file (眨眼). - -To complete the process, lets see if we can retrieve the metadata. Run the query command again: -$ cd /usr/local/oodt/cas-filemgr/bin -$ ./query-tool --url http://localhost:9000 --sql -query 'SELECT * FROM GenericFile' - -The output should look like: -Sep 16, 2011 2:21:54 PM org.apache.oodt.cas.filemgr.system.XmlRpcFileManager complexQuery -INFO: Query returned 1 results -/var/archive/data/blah.txt,GenericFile,blah.txt,blah.txt,2011-09-16T14:09:43.405+02:00,c2fbf4b9-e05c-11e0-9022-77a707615e7f,Flat,text/plain,text,plain - -Check to see if the file has appeared in the archive: -$ ls /var/archive/data/blah.txt/ -blah.txt - -Query commands do not depend on the underlying catalog implementation. The --sql and --lucene instead describe the filemgr query syntax. - -At the time of writing this tutorial, composing queries using query-tool is not entirely straight forward, but entirely usable. Formatting of these queries is critical, small deviations from the syntax can result in the query return an unexpected value or throwing an exception. - -Some things to note about SQL queries: - -Use double quotes ("") for when specifying the SQL syntax. The single quote ('') is used for string values in a WHERE clause, e.g WHERE Filename='blah.txt' -Count the number of -- before each command line option. Some are -- and others are -. -The order of the return values for a search is not guaranteed unless you specify the \outputFormat option. -Here is a somewhat verbose example that uses all the SQL-like syntax that I am currently aware of (apologies for all the line breaks). - -$ cd /usr/local/oodt/cas-filemgr/bin -$ ./query-tool --url http://localhost:9000 --sql \ --query "SELECT CAS.ProductReceivedTime,CAS.ProductName,CAS.ProductId,ProductType,\ -ProductStructure,Filename,FileLocation,MimeType \ -FROM GenericFile WHERE Filename='blah.txt'" -sortBy 'CAS.ProductReceivedTime' \ --outputFormat '$CAS.ProductReceivedTime,$CAS.ProductName,$CAS.ProductId,$ProductType,\ -$ProductStructure,$Filename,$FileLocation,$MimeType' -The output should look like: -2011-10-07T10:59:12.031+02:00,blah.txt,a00616c6-f0c2-11e0-baf4-65c684787732, -GenericFile,Flat,blah.txt,/var/kat/archive/data/blah.txt,text/plain - -Now you can also check out some of the other 12 --operation possibilities for filemgr-client. For instance: - -$ ./filemgr-client --url http://localhost:9000 --operation --hasProduct --productName blah.txt - -Or: - -$ ./filemgr-client --url http://localhost:9000 --operation --getFirstPage --productTypeName GenericFile - -A few more tools -Cameron Goodale has written some useful command line tools aliases that are worth mentioning before we continue. See the following two web pages: https://issues.apache.org/jira/browse/OODT-306 -BASH and TCSH shell tools for File Manager - -Tips and Tricks for FileManager -Q: My Lucene Index Catalog is running slow now that I have over 100,000 products cataloged. How can I get the speed back? - -A: Run this command: -java -Djava.endorsed.dirs= org.apache.oodt.cas.filemgr.tools.OptimizeLuceneCatalog --catalogPath \ No newline at end of file diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/OODT Filemgr User Guide.txt.xml.xls b/src/main/resources/sdtocode/doc/Apache OODT File Manager/OODT Filemgr User Guide.txt.xml.xls deleted file mode 100644 index ff0881c3ee525ca8c15dd4d5aad253cf6dbbfafa..0000000000000000000000000000000000000000 Binary files a/src/main/resources/sdtocode/doc/Apache OODT File Manager/OODT Filemgr User Guide.txt.xml.xls and /dev/null differ diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Package org.apache.oodt.cas.filemgr.cli.action-relation.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/Package org.apache.oodt.cas.filemgr.cli.action-relation.txt deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Package org.apache.oodt.cas.filemgr.cli.action.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/Package org.apache.oodt.cas.filemgr.cli.action.txt deleted file mode 100644 index 1c22f3309dfbbc720b09c3f5d59af9540e8c183b..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Package org.apache.oodt.cas.filemgr.cli.action.txt +++ /dev/null @@ -1,26 +0,0 @@ -Catalog and Archive File Management Component 0.12 API -Packages -Package Description -org.apache.oodt.cas.filemgr.catalog -org.apache.oodt.cas.filemgr.catalog.solr -org.apache.oodt.cas.filemgr.cli.action -org.apache.oodt.cas.filemgr.datatransfer -org.apache.oodt.cas.filemgr.exceptions -org.apache.oodt.cas.filemgr.ingest -org.apache.oodt.cas.filemgr.metadata -org.apache.oodt.cas.filemgr.metadata.extractors -org.apache.oodt.cas.filemgr.metadata.extractors.examples -org.apache.oodt.cas.filemgr.repository -org.apache.oodt.cas.filemgr.structs -org.apache.oodt.cas.filemgr.structs.exceptions -org.apache.oodt.cas.filemgr.structs.query -org.apache.oodt.cas.filemgr.structs.query.conv -org.apache.oodt.cas.filemgr.structs.query.filter -org.apache.oodt.cas.filemgr.structs.type -org.apache.oodt.cas.filemgr.structs.type.examples -org.apache.oodt.cas.filemgr.system -org.apache.oodt.cas.filemgr.system.auth -org.apache.oodt.cas.filemgr.tools -org.apache.oodt.cas.filemgr.util -org.apache.oodt.cas.filemgr.validation -org.apache.oodt.cas.filemgr.versioning \ No newline at end of file diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Package org.apache.oodt.cas.filemgr.cli.action.txt.xml.xls b/src/main/resources/sdtocode/doc/Apache OODT File Manager/Package org.apache.oodt.cas.filemgr.cli.action.txt.xml.xls deleted file mode 100644 index f8af3037fb4043ef401be08ca2007838c77a1e74..0000000000000000000000000000000000000000 Binary files a/src/main/resources/sdtocode/doc/Apache OODT File Manager/Package org.apache.oodt.cas.filemgr.cli.action.txt.xml.xls and /dev/null differ diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/React file manager-relation.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/React file manager-relation.txt deleted file mode 100644 index 08f288bc70551a07ef06090422da9acb09958dbd..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/doc/Apache OODT File Manager/React file manager-relation.txt +++ /dev/null @@ -1,770 +0,0 @@ -your&theme& -set&new feature&AGGREGATION -potential j and css ) ideal&HTML5 and Bootstrap&AGGREGATION -component&dashboard template&AGGREGATION -Redux&application&依赖 -It&beautiful design&依赖 -LOT&components and feature&AGGREGATION -you&live preview&依赖 -it&LOT&依赖 -it&components and feature&依赖 -filemanager namespace&FileManagerCommand class&依赖 -Module file&multi-pane module editor&依赖 -Module file&design manager&依赖 -include&App&依赖 -include&ej2-react-filemanager package&依赖 -include&FileManagerComponent&依赖 -1&4&依赖 -1&Bootstrap version&依赖 -1&Bootstrap version&依赖 -1&4&依赖 -download manager&HTTP connection&依赖 -I&shortcode&依赖 -Mobile application&greater value&依赖 -we&package&依赖 -we&package&依赖 -Mobile application&business&依赖 -their&website& -Mobile application&mobile website&依赖 -Storybook&setup& -your&self& -You&webpack setup&依赖 -your&opinions& -webpack setup&setup&GENERALIZATION -import&dx& -import React&React&GENERALIZATION -your&needs& -dependency&itself&依赖 -you&file upload component&依赖 -top&list&AGGREGATION -order&file&AGGREGATION -your&files& -You&file&依赖 -You&order&依赖 -react-overlay&them&依赖 -you&them&依赖 -aspect&target&AGGREGATION -net core suite&100 + fully-featured UI component&依赖 -net core suite&100 + fully-featured UI component&依赖 -net core suite&100 + fully-featured UI component&依赖 -You&file or folder&依赖 -left&file or folder&AGGREGATION -You&left&依赖 -You&symbol&依赖 -its&properties& -warranty&kind&AGGREGATION -heart&smartphone&AGGREGATION -file manager application&smartphone&依赖 -rest&page&AGGREGATION -" trap " focus&keyboard navigation cycle&依赖 -Modal&focus& -Overview&Kendo UI FileManager&AGGREGATION -I&react&依赖 -your&collaboration& -my&tasks& -your&JavaScript& -your&url& -current url&Drupal&AGGREGATION -Syncfusion&Studio& -our&model& -file&a dot )&依赖 -your&manager& -dot file&file&GENERALIZATION -file&file manager&依赖 -it&hidden file&依赖 -you&dot file&依赖 -autoFocus&modal&依赖 -autoFocus&vanilla Bootstrap&依赖 -sample&Amazon S3 file system provider&依赖 -React&reusable UI component&依赖 -React&us&依赖 -SVN&web URL&依赖 -you&look&依赖 -you&file input&依赖 -look&file input&AGGREGATION -you&form&依赖 -your&design& -Chocolatey&business&依赖 -Chocolatey&software deployment&依赖 -default image const [ defaultuserimage react filemanager&Specify&依赖 -svg '&file input const imageref = useref&依赖 -default image const [ defaultuserimage react filemanager&Specify&依赖 -default image const [ defaultuserimage react filemanager&default image const [ defaultuserimage react filemanager&依赖 -default image const [ defaultuserimage react filemanager&default image const [ defaultuserimage react filemanager&依赖 -svg '&file input const imageref = useref&依赖 -your&path& -FTP Access Upload file&easier and faster way to upload and download&依赖 -FTP Access Upload file&easier and faster way to upload and download&依赖 -default locale&file manager&AGGREGATION -our&manager& -It&kendo-react-upload package&依赖 -It&NPM&依赖 -It&standalone app&依赖 -Python dictionary&key&依赖 -Python dictionary&performance&依赖 -React ’s implementation&popular drag and drop library&AGGREGATION -react-dropzone&file uploading&实现 -react-dropzone&popular drag and drop library&实现 -our&opinion& -Webix library&opinion&依赖 -Webix library&best solution&依赖 -we&filemanager (&依赖 -we&File Server Node API component&依赖 -we&opuscapita )&依赖 -your&application& -You&span&依赖 -You&grid element&依赖 -folder base file browser&folder base file browser&依赖 -folder base file browser&folder base file browser&依赖 -folder base file browser&folder base file browser&依赖 -folder base file browser&folder base file browser&依赖 -folder base file browser&flat keyed list&依赖 -folder base file browser&folder base file browser&依赖 -folder base file browser&flat keyed list&依赖 -folder base file browser&flat keyed list&依赖 -folder base file browser&folder base file browser&依赖 -folder base file browser&folder base file browser&依赖 -folder base file browser&folder base file browser&依赖 -folder base file browser&flat keyed list&依赖 -folder base file browser&folder base file browser&依赖 -folder base file browser&flat keyed list&依赖 -flat keyed list&object&AGGREGATION -folder base file browser&flat keyed list&依赖 -folder base file browser&folder base file browser&依赖 -folder base file browser&flat keyed list&依赖 -folder base file browser&folder base file browser&依赖 -folder base file browser&flat keyed list&依赖 -folder base file browser&folder base file browser&依赖 -folder base file browser&flat keyed list&依赖 -folder base file browser&flat keyed list&依赖 -folder base file browser&flat keyed list&依赖 -folder base file browser&flat keyed list&依赖 -client react connector&Google Drive API v2&依赖 -client react connector&Google Drive API v2&依赖 -scaffold&what&依赖 -You&project&依赖 -you&right&依赖 -ES7 syntax Another file format&MD file extension&依赖 -specify camera icon&button text import camera&依赖 -React Native This&lightweight and easy-to-use password manager Clonezilla&依赖 -exact mirror&React Native project&AGGREGATION -React Native This&React Native project&依赖 -operating system&file manager&依赖 -operating system&required file&依赖 -Create React App&React Application&依赖 -webpack configuration&you&依赖 -Storybook&configuration& -their&files& -customer&static file&依赖 -one&admin tool&AGGREGATION -customer&shared host&依赖 -our&customers& -null filebrowser dialog&FileBrowser object&依赖 -let ’s&Bootstrap and React&依赖 -My&folders& -My&Documents& -You&Media&依赖 -You&file&依赖 -your&React& -flag add&dependency&依赖 -one&two way&AGGREGATION -Run bower&&依赖 -Run bower&save&依赖 -flag add&dependency&依赖 -partition and disk&React debugging tool&依赖 -partition and disk&Chrome Developer Tools&依赖 -FileBrowser dialog&object previewer/property manager&依赖 -FileBrowser dialog&FileBrowser object&依赖 -them&individual contributor role&依赖 -them&manager role&依赖 -json&JSON&依赖 -It&collaboration&依赖 -It&my development task and time&依赖 -It&me&依赖 -template and NOT&final purchase file&AGGREGATION -part&template and NOT&AGGREGATION -They&template and NOT&依赖 -file&above import&依赖 -right click Libraries Add&Go&依赖 -project&Go& -your&project& -an html5 drag & drop file uploader&an html5 drag & drop file uploader&依赖 -I&somebody&依赖 -you&‘ public_html ’ folder&依赖 -you&domain&依赖 -you&file manager&依赖 -Your&first& -build script ( dev-build&build script ( dev-build&依赖 -Manager&icons& -) wrap long file name&Folder Tree&依赖 -) wrap long file name&icon&依赖 -) wrap long file name&) wrap long file name&依赖 -com&working&依赖 -com&you&依赖 -com&opportunity&依赖 -You&flexible ReactJS component&依赖 -create-react-app&files [ 0 ] ) }&依赖 -create-react-app&browser&依赖 -list&top react native developer&AGGREGATION -I&File Manager&依赖 -I&file&依赖 -popular open-source JavaScript library&you&AGGREGATION -project&build&依赖 -part&project&AGGREGATION -project&part&依赖 -project&restore&依赖 -json file&Build ”&依赖 -json file&“ Enable Restore&依赖 -json file&Build ”&依赖 -json file&“ Enable Restore&依赖 -lot&people&AGGREGATION -file&module-name&依赖 -upload react website&“&AGGREGATION -subdomain open file manager&new folder&依赖 -upload react website&” folder&依赖 -I&Kendo UI File Manager&依赖 -I&custom command&依赖 -Restore&client-side library&依赖 -Restore&libman&依赖 -Restore&libman&依赖 -Restore&client-side library&依赖 -user&own directory&依赖 -It&multiple user&依赖 -creation&multiple user&AGGREGATION -It&creation&依赖 -its&directory& -we&task manager application&依赖 -it&previewer&依赖 -DevExtreme JavaScript FileManager component&you&依赖 -DevExtreme JavaScript FileManager component&files and directory&依赖 -I&addition&依赖 -I&React JSX file&依赖 -corresponding online repository&package&AGGREGATION -time&top&依赖 -time&GCS cloud storage bucket&依赖 -top&GCS cloud storage bucket&AGGREGATION -time&Dropbox-like functionality&依赖 -time&Dropbox-like functionality&依赖 -time&GCS cloud storage bucket&依赖 -time&top&依赖 -you&standard file input HTML element&依赖 -Binding&UI element&AGGREGATION -React&modern web application&依赖 -one&best choice&AGGREGATION -React&best choice&依赖 -React&slim apus&依赖 -installation file&it&AGGREGATION -KendoReact Upload component&KendoReact library&依赖 -KendoReact library&React UI component&AGGREGATION -KendoReact Upload component&React UI component&依赖 -part&KendoReact library&AGGREGATION -your&analytics& -you&analytic&依赖 -React component&component&GENERALIZATION -new React component&JavaScript and TypeScript&依赖 -file caching system&two main part&依赖 -Our&system& -tailor fman&powerful plugin system&依赖 -tailor fman&powerful plugin system&依赖 -its&system& -tailor fman&powerful plugin system&依赖 -tailor fman&powerful plugin system&依赖 -tailor fman&powerful plugin system&依赖 -tailor fman&powerful plugin system&依赖 -tailor fman&powerful plugin system&依赖 -tailor fman&powerful plugin system&依赖 -React&single button&依赖 -app&interface& -React&few piece&依赖 -React&interface&依赖 -few piece&interface&AGGREGATION -requirement&File Upload Component&依赖 -requirement&React&依赖 -file manager&local computer&依赖 -npm&save&依赖 -React&web&依赖 -React&frontend library&依赖 -your&computer& -npm&react-file&依赖 -reacto&x86/x64 personal computer&依赖 -npm start&build&依赖 -npm start&a new&依赖 -system service&long-running download&依赖 -system service&handling&依赖 -system service&long-running download&依赖 -system service&handling&依赖 -handling&long-running download&AGGREGATION -RNFetchBlob&functionality& -method&you&依赖 -method&Windows Explorer&依赖 -its&features& -list&key feature&AGGREGATION -md file store transaction&md file store transaction&依赖 -second be&Redux store&依赖 -set&actions and reducer&AGGREGATION -second be&Redux store&依赖 -second be&actions and reducer&依赖 -Webix widgets and application&Angular or React environment&依赖 -integrate tinymce editor&Laravel&依赖 -integrate tinymce editor&image upload jquery php july 27 , 2019 2,006 view&依赖 -integrate tinymce editor&image upload jquery php july 27 , 2019 2,006 view&依赖 -integrate tinymce editor&image upload jquery php july 27 , 2019 2,006 view&依赖 -integrate tinymce editor&Laravel&依赖 -integrate tinymce editor&image upload jquery php july 27 , 2019 2,006 view&依赖 -integrate tinymce editor&image upload jquery php july 27 , 2019 2,006 view&依赖 -integrate tinymce editor&Laravel&依赖 -integrate tinymce editor&image upload jquery php july 27 , 2019 2,006 view&依赖 -integrate tinymce editor&image upload jquery php july 27 , 2019 2,006 view&依赖 -integrate tinymce editor&image upload jquery php july 27 , 2019 2,006 view&依赖 -integrate tinymce editor&image upload jquery php july 27 , 2019 2,006 view&依赖 -integrate tinymce editor&Laravel&依赖 -integrate tinymce editor&image upload jquery php july 27 , 2019 2,006 view&依赖 -integrate tinymce editor&image upload jquery php july 27 , 2019 2,006 view&依赖 -integrate tinymce editor&image upload jquery php july 27 , 2019 2,006 view&依赖 -image upload jquery php july 27 , 2019 2,006 view&Sortable&依赖 -integrate tinymce editor&Laravel&依赖 -integrate tinymce editor&Laravel&依赖 -we&animation&依赖 -we&File Manager App uus&依赖 -we&React Native&依赖 -You&high end protection&依赖 -It&file manager&依赖 -you&React&依赖 -you&reusable component&依赖 -table&file manager&依赖 -table&default texts and message&依赖 -table&en culture&依赖 -default texts and message&file manager&AGGREGATION -en culture&culture&GENERALIZATION -modern toolchain&React developer&AGGREGATION -this project base course&this project base course&依赖 -user&image and video and etc and Instagram&依赖 -user&example&依赖 -repo&page& -your&repo& -your&repository& -benefit&React&AGGREGATION -folder&creation&GENERALIZATION -full list&compatible frameworks and integration example&AGGREGATION -Files and folder&npm install&依赖 -part&default react-native init installation&AGGREGATION -other file & folder and some truncate )&default react-native init installation&依赖 -our&focus& -other file & folder and some truncate )&default react-native init installation&依赖 -you&other file&依赖 -Accessible&Keyboard&依赖 -Accessible&Keyboard&依赖 -own React&Browserify&依赖 -own React&process (&依赖 -npm&package manager&依赖 -npm&Node&依赖 -multiple package&package&依赖 -Bower&jQuery&依赖 -multiple package&example&依赖 -You&two option&依赖 -You&file uploader&依赖 -reacto&computer&依赖 -your&tools& -free , opensource reimplementation&windows relate&AGGREGATION -Paper Kit reacto&windows relate&实现 -Developer Express Inc&warranty&依赖 -warranty&merchantability and fitness&AGGREGATION -Developer Express Inc&warranty&依赖 -Removed complementary design file&[ 5&依赖 -tagspace feature basic file management operation&tagspace feature basic file management operation&依赖 -it&simple file manager&依赖 -dhtmlxGrid&rich API functionality&依赖 -unparalleled array&feature&AGGREGATION -Asus file manager&file manager&GENERALIZATION -part&dhtmlxSuite library&AGGREGATION -world&grid& -last post 2 hour&last post 2 hour&依赖 -net core&RSS 0&依赖 -last post 2 hour&last post 2 hour&依赖 -last post 2 hour&fiazahmed an electron base file manager&依赖 -net core&RSS 0&依赖 -net core&RSS 0&依赖 -last post 2 hour&fiazahmed an electron base file manager&依赖 -It&visual file manager&依赖 -it&function&依赖 -your&app& -set&function&AGGREGATION -it&set&依赖 -presence&drag & drop responsiveness&依赖 -presence&handler&AGGREGATION -newer version&program use&AGGREGATION -FileManager&file system provider&依赖 -you&opinion&依赖 -you&folder&依赖 -fastlane&folder&依赖 -you&file&依赖 -you&file&依赖 -Firebase issue&issue&GENERALIZATION -Firebase issue&[ 5&依赖 -Firebase issue&[ 5&依赖 -create-react-app utility&tool&依赖 -create-react-app utility&Babel and webpack&依赖 -create-react-app utility&Babel and webpack&依赖 -create-react-app utility&tool&依赖 -we&Client React component&依赖 -Client React component&React component&GENERALIZATION -our&library& -our&detail& -nyc_output and coverage folder&instrumentation detail&依赖 -React admin dashboard&frontend framework&依赖 -React admin dashboard&material-uus&依赖 -term&frontend framework&AGGREGATION -React admin dashboard&term&依赖 -It&accessing&依赖 -It&user&依赖 -It&common file operation&依赖 -sample&Android 4 ICS emulator image&依赖 -smartphone&file manager&依赖 -smartphone&view&依赖 -npm package&call package&依赖 -npm package&file&依赖 -you&number&依赖 -see bundler default&full list&依赖 -see bundler default&full list&依赖 -Bower&front-end&依赖 -It&file&依赖 -Cezerin&open-source ecommerce platform&依赖 -your&component& -you&Social Sites&依赖 -your&site& -9 all file upload&a json response (&依赖 -9 all file upload&a json response (&依赖 -curriculum&introspection&依赖 -curriculum&building skill&依赖 -curriculum&addition&依赖 -Plugin&Front-end & Back-end&依赖 -Plugin&two part&依赖 -re-run&package manager&AGGREGATION -your&changes& -user&upload button&依赖 -this package help&file and assest&依赖 -this package help&you&依赖 -your&service& -you&large files/streaming&依赖 -you&Android Download Manager&依赖 -help&Suite component&AGGREGATION -React component&yarn or npm&依赖 -you&yourself&依赖 -you&few step&依赖 -app&file& -class App&App&GENERALIZATION -class App&React&依赖 -use npm&Expo CLI command line utility&依赖 -you&package&依赖 -html file&code coverage&依赖 -html file&code coverage&依赖 -our&coverage& -bootstrap npm&axio&依赖 -react-toastify&beautiful notification&依赖 -We&react project react-file-upload – cd react-file-upload&依赖 -bootstrap npm&npm&GENERALIZATION -bootstrap help&uus&依赖 -CKEditor 4&external file manager and ( file browser/uploader&依赖 -CKEditor 4&( file browser/uploader&依赖 -smart filemanager&Material Design style&依赖 -smart filemanager&Jonas Sciangula Street&依赖 -smart filemanager&Material Design style&依赖 -smart filemanager&Material Design style&依赖 -smart filemanager&angularj&依赖 -smart filemanager&Jonas Sciangula Street&依赖 -smart filemanager&angularj&依赖 -smart filemanager&Jonas Sciangula Street&依赖 -smart filemanager&angularj&依赖 -page ( module instance&page )&依赖 -file&page )&依赖 -file&page&依赖 -file&individual rendered module&依赖 -instance&module&AGGREGATION -your&server& -website&page& -your&website& -You&custom thumbnail and text description&依赖 -You&file or folder&依赖 -Drag and Drop Support&files or folder&依赖 -React FileManager component 23 Feb 2021 / 1 minute&file manager&依赖 -your&program& -you&it&依赖 -your&Panel& -you&folder&依赖 -Panel&manager& -file&actual file&依赖 -its&details& -file&actual file&依赖 -rest¬hing ( white page )&依赖 -rest&website&AGGREGATION -React File Manger Multi-column File Manager&react-beautiful-dnd&依赖 -js developers one&js developers one&依赖 -React and TypeScript&development&实现 -development&visual interface&AGGREGATION -React and TypeScript&visual interface&实现 -/ /&path / / Specify&依赖 -/ /&it&依赖 -/ /&/&GENERALIZATION -Specify&defaultUser& -your&defaultUser& -your&Specify& -software management automation&installer&依赖 -Chocolatey&Windows&依赖 -Files file&cloud service&依赖 -Files file&server&依赖 -a flexible and beautiful select input control&reactj&依赖 -a flexible and beautiful select input control&reactj&依赖 -a flexible and beautiful select input control&reactj&依赖 -a flexible and beautiful select input control&multiselect , autocomplete and ajax support&依赖 -a flexible and beautiful select input control&multiselect , autocomplete and ajax support&依赖 -a flexible and beautiful select input control&multiselect , autocomplete and ajax support&依赖 -a flexible and beautiful select input control&reactj&依赖 -a flexible and beautiful select input control&multiselect , autocomplete and ajax support&依赖 -a flexible and beautiful select input control&multiselect , autocomplete and ajax support&依赖 -a flexible and beautiful select input control&multiselect , autocomplete and ajax support&依赖 -a flexible and beautiful select input control&multiselect , autocomplete and ajax support&依赖 -a flexible and beautiful select input control&multiselect , autocomplete and ajax support&依赖 -assetExts resolver option&package&AGGREGATION -Metro Bower keeps track&package&AGGREGATION -You&other type&依赖 -your&track& -You&support&依赖 -its&content& -please note&folder&依赖 -you&file&依赖 -you&blank file&依赖 -their&systems& -KendoReact Upload&file system&依赖 -KendoReact Upload&file&依赖 -Template&require syntax&依赖 -uploaded file&Java SDK • fix&依赖 -their&checker& -uploaded file&http status 404-not found "&依赖 -uploaded file&upload image&依赖 -uploaded file&java sdk • fixed unable&依赖 -kendo ui r1&kendo&依赖 -kendo ui r1&kendo ui r1&依赖 -npm&g expo-cli Use Expo&依赖 -npm&React Native app&依赖 -component&Url&依赖 -component&file system item&依赖 -we&dependency&依赖 -package babel-plugin-styled-component&documentation&依赖 -your&connector& -you&it&依赖 -you&what&依赖 -our&demo& -you&it&依赖 -you&what&依赖 -container folder script = > react folder&file&依赖 -container folder script = > react folder&file&依赖 -container folder script = > react folder&new item dialog popup&依赖 -container folder script = > react folder&new item dialog popup&依赖 -container folder script = > react folder&new item dialog popup&依赖 -container folder script = > react folder&new item dialog popup&依赖 -container folder script = > react folder&file&依赖 -container folder script = > react folder&file&依赖 -node package manager ( npm )&web developer tool belt&依赖 -one&most demanded gadget&AGGREGATION -node package manager ( npm )&most demanded gadget&依赖 -You&Task Manager&依赖 -You&2 option&依赖 -async function&webpack config&依赖 -we&way&依赖 -we&massive e-commerce application&依赖 -Free Web File Thunar&Benedikt Meurer&依赖 -Xfce&manager& -Ignite UI&most complete Microsoft Excel solution&依赖 -Ignite UI&most complete Microsoft Excel solution&依赖 -its&own& -png extension ) Let&quick look&依赖 -you&" Task Manager " app&依赖 -Work '&export const fileitem = [ { ' name '&依赖 -Custom File System Provider&custom api&实现 -Custom File System Provider&you&实现 -export const fileitem = [ { ' name '&export const fileitem = [ { ' name '&依赖 -Work '&[ { ' name '&依赖 -{&name& -it&click&依赖 -it&file manager&依赖 -Webix&ready-made solution&依赖 -I&Laravel + React&依赖 -I&demo&依赖 -We&TypeScript&依赖 -We&website&依赖 -We&Gatsby&依赖 -our&CLI& -we&what&依赖 -You&Dropzone component&依赖 -Dropzone component&component&GENERALIZATION -your&domain& -child component&you application&AGGREGATION -many way&programming problem&依赖 -many way&programming problem&依赖 -we&file (&依赖 -we&first component&依赖 -npm package&package&GENERALIZATION -One common way&CSS&依赖 -yarn&react-dom yarn&依赖 -React&component&GENERALIZATION -article&simple way&依赖 -article&approach&实现 -You&custom color&依赖 -You&folder and tag&依赖 -net /&/&依赖 -we&Laravel ’s Storage API&依赖 -you&essential j 2 file manager component&依赖 -you&application&依赖 -all basic file handling mechanism&all basic file handling mechanism&依赖 -all basic file handling mechanism&all basic file handling mechanism&依赖 -all basic file handling mechanism&upload , download , read , edit , delete , search&依赖 -all basic file handling mechanism&upload , download , read , edit , delete , search&依赖 -FileManager UI component&file system&依赖 -com&team&依赖 -com&delhi/ncr ( india ) employment&依赖 -com&delhi/ncr ( india ) ctc&依赖 -our&team& -com&React JS Developers&依赖 -Permanent Employment Place&Work&AGGREGATION -component&provider& -React&code convention&依赖 -React&strict rule&依赖 -Redux side&thing&AGGREGATION -site&different source&依赖 -we&orthodox WEB file manager&依赖 -analogue&ncdu )&AGGREGATION -we&idea&依赖 -we&working&依赖 -server&site& -lot&other great stuff&AGGREGATION -file upload form&traditional HTML site&依赖 -file upload form&page refresh&依赖 -Chocolatey&w/SCCM&依赖 -React File Manager component&easy uploading and downloading&依赖 -easy uploading and downloading&file&AGGREGATION -React File Manager component&file&依赖 -React File Manager component&File Manager component&GENERALIZATION -React File Manager component&file in ###&依赖 -I&a page “ commercial file&依赖 -I&file&依赖 -I&example&依赖 -I&shortcode correspond&依赖 -I&shortcode correspond&依赖 -folder&file&AGGREGATION -React&software engineer&依赖 -React&Jordan Walke&依赖 -React&working&依赖 -npm&npm run start Design&依赖 -I&react&实现 -I&simple file explorer&依赖 -one&file&AGGREGATION -your&results& -compiler strip&process&依赖 -signature&type&AGGREGATION -compiler strip&function and method body&依赖 -compiler strip&process&依赖 -compiler strip&function and method body&依赖 -You&sub-folder&依赖 -React Filemanager Hello ex angular-filemanager user and&React&依赖 -It&laborasyon&依赖 -It&ThemeForest&依赖 -we&image&依赖 -we&tutorial&依赖 -we&react js component&依赖 -you&optional Yarn package manager&依赖 -File Manager component&files and folder&依赖 -File Manager component&multiple selection&依赖 -File Manager component&files and folder&依赖 -multiple selection&files and folder&AGGREGATION -File Manager component&multiple selection&依赖 -a ready-made set&icon&AGGREGATION -It&beautiful design&依赖 -we&80 react widget and plugin&依赖 -string index&sequence – Python&依赖 -string index&array element&依赖 -it&live change&依赖 -issue&thing&依赖 -issue&lot&依赖 -issue&thing&依赖 -issue&lot&依赖 -lot&thing&AGGREGATION -this package support multiple file selection&cloud storage integration&依赖 -this package support multiple file selection&this package support multiple file selection&依赖 -this package support multiple file selection&this package support multiple file selection&依赖 -this package support multiple file selection&cloud storage integration&依赖 -least two field&name and version&依赖 -least two field&definition file&依赖 -definition file&file&GENERALIZATION -new bordered & dark layout new ecommerce dashboard invoice bamburgh react admin dashboard&React&依赖 -0 ] – 2020-11-28 HTML , HTML + Laravel ADDED All-new design&new bordered & dark layout new ecommerce dashboard invoice bamburgh react admin dashboard&依赖 -0 ] – 2020-11-28 HTML , HTML + Laravel ADDED All-new design&Reactstrap PRO&依赖 -APP&green circle button&依赖 -my&APP& -we&dependency&依赖 -we&npx&依赖 -Download Epic React – HR Management Admin Template&commercial use&依赖 -item&you&依赖 -Download Epic React – HR Management Admin Template&download link&依赖 -Download Epic React – HR Management Admin Template&developer puffintheme&依赖 -Minor fix&RTL SCSS&AGGREGATION -npm&react-dom npm install&依赖 -your&Fodor& -It&React framework&依赖 -File Manager&upload&依赖 -File Manager&new file&依赖 -browser&it&依赖 -example&integration usage&AGGREGATION -you&example&依赖 -you&integration usage&依赖 -It&large UI collection&依赖 -import React&import { filemanager , filenavigator }&依赖 -import React&react-filemanager&依赖 -/&react-filemanager&GENERALIZATION -FileManager&operation&依赖 -control&Telerik UI&依赖 -control&ASP&依赖 -part&Telerik UI&AGGREGATION -api&azure-file-system&依赖 -api&client&实现 -api&access&实现 -api&Azure Blob Storage&实现 -banzay/friends-app-redux second take&friends app&依赖 -Excel Viewer widget&) edition webix community ( opensource&依赖 -NET Core FileManager&user&依赖 -your&users& -their&applications& -domain&‘ public_html ’ folder&依赖 -Your&domain& -domain&top ‘ index&依赖 -folder&required React Native iOS bridge&依赖 -You&folder&依赖 -folder&single framework&依赖 -you&Table service&依赖 -you&state information&依赖 -web-based gaming experience&two part&依赖 -its&core& -file manager&culture&依赖 -Predefined connector&Server Node API v1 Localization&依赖 -Predefined connector&React FileManager component&依赖 -texts and message&file manager&AGGREGATION -babelrc file&present&依赖 -Cloudinary&it&依赖 -DOM&file ( eg&依赖 -File Manager&you&依赖 -content&package&AGGREGATION -basic file operation&file manager component&依赖 -basic file operation&file manager component&依赖 -all later&Pocket&依赖 -your&client& -your&grid& -part&UI framework&AGGREGATION -component&native iOS/Android file viewer&依赖 -our&application& -we&file&依赖 -we&file picker&依赖 -I&laravel application&依赖 -file&content& -I&file manager package&依赖 -I&you&依赖 -Install Step 1 npm i react-native-azure-blob-storage-manager&Manual Installation Installation&依赖 -We&react-dropzone&依赖 diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/React file manager.txt b/src/main/resources/sdtocode/doc/Apache OODT File Manager/React file manager.txt deleted file mode 100644 index 600b501136a12f398d7b5d1d04cd146a9e6d4d6e..0000000000000000000000000000000000000000 --- a/src/main/resources/sdtocode/doc/Apache OODT File Manager/React file manager.txt +++ /dev/null @@ -1 +0,0 @@ -react file manager All components included in this dashboard template has been developed to bring all the potential of HTML5 and Bootstrap plus a set of new features (JS and CSS) ideal for your next dashboard admin theme or admin web application project. 확장성을 보유할것 외부 프로젝트에서도 Description. Redux helps you write applications that behave consistently, run in different environments (client, server, and native), and are easy to test. It has a beautiful design, as you can see from the live previews and it contains a LOT of components and features. filemanager namespace exposes the FileManagerCommand class that could be extended to implement a custom File Manager command. Just display a list within 2 predifined tabs (folders). Create React App - TS docs; Next. The Edit screen with option to select one or more files is displayed. Add Start script to package. 9,676 4. Inbuilt Search textbox in FileManager: See Also. Module files are represented in the design manager in a multi-pane module editor. To include the File Manager component in application import the FileManagerComponent from ej2-react-filemanager package in App. 1 - 28 November 2020 ----- - Upgraded Bootstrap version to 4. Filemanager with React & Nodejs . prod. The download manager handles HTTP connections, monitors connectivity changes, reboots, and ensures each download completes successfully. I would like this shortcode to be dynamic, i. Grab the demo from Github if you haven't done this yet. Mobile applications definitely offer a greater value to businesses than their mobile website In this tutorials we will use a package named @react-native-community/checkbox to add checkboxes in react native. Express your opinions freely and help others including your future self You can customize Storybook's webpack setup by providing a webpackFinal field in . 4. 3. import React from 'react'; import 'devextreme/dist/css/dx. js, Express and TypeScript. Install the React components and choose a theme that suits your needs. target. 2 - Ability to translate Wireframes and PSD Designs into functional web apps using HTML5, React , Node. In this tutorial you will learn how to create a working file upload component with react from scratch using no dependencies other than react itself. 2/6. /. You can rearrange the order of your files by dragging them around to move the important files to the top of the list for faster access. Run the Drupal Page having React Nested modals aren’t supported, but if you really need them the underlying react-overlays can support them if you're willing. Download the corresponding App Center SDK for iOS frameworks provided as a zip file and unzip it. NET Core suite along with 100+ fully-featured UI components designed to speed up delivery & improve every aspect of target. You’ll see a plus symbol to the left of the file or folder. com and its affiliated web properties is provided "as is" without warranty of any kind. An electron based file manager. The file manager application is like the heart of a smartphone. Modal's "trap" focus in them, ensuring the keyboard navigation cycles through the modal, and not the rest of the page. Async uploading with AJAX, or encode files as base64 data and send along form post. Overview of Kendo UI FileManager; Sort in Kendo UI FileManager; Toolbar Commands in Kendo UI FileManager Express your opinions freely and help others including your future self I am a beginner in react. LibraryManager. Developed with the latest jQuery plugins. html and . Complete file and folder manager: Create, rename, move and delete a folder. It's very important for me your collaboration on my development tasks and time. Test your JavaScript, CSS, HTML or CoffeeScript online with JSFiddle code editor. Go to react_code\src and change the apiUrl inside config. js as per your current url of Drupal. rtl8761a_mp_chip_bt40_fw_asic_rom_patch_8192ee_new. Say “EDIT MODE”. 1. To enable profiling in production mode, modify Webpack configuration file (config/webpack. To download and start utilizing Syncfusion's Essential Studio for React components, see our pricing model. A simple file manager built with react. azurewebsites. 0. Vue. Updated laravel 7 to all full version and starter-kit; React Fixed. mp4 videos to the server. Learn to build modern web applications using Angular, React & Vue! File Upload Component with Vue. A file input (dropzone) management component for React. TIP: If you have never seen a dot file (a file starting with a dot) it might be odd at first because that file might not appear in your file manager, as it’s a hidden file. Unlike vanilla Bootstrap, autoFocus works in Modals because React handles the implementation Free download Filedash – File Manager Dashboard Nulled. Maybe later i can have a button to have the grid view File-Manager 개발정의서 들어가며 본 문서는 인수인계 목적이 아닌 개발이 완료된 제품에 대한 이해를 돕기위해 제작된 개발 정의서입니다. Say for instance that you want to open the file select dialogue for a user to select an file to upload. This sample demonstrates how to utilize the Amazon S3 file system provider to manage the files in File Manager component. Deploy Trillo File Manager from GCP Marketplace In this tutorials we will use a package named @react-native-community/checkbox to add checkboxes in react native. Choose a device definition, Nexus 5X is suggestable. Create a new project with React Native. React also allows us to create reusable UI components. Use Git or checkout with SVN using the web URL. bs4 File Manager. v6. js . Source code: https://bit. Learn more . log(event. Also, you might want to customize the look of the file input in the form to make it resonate with your overall app design. Chocolatey is trusted by businesses to manage software deployments. Conclusion Let’s work that out. Scheduler. svg'; // replace it with your path // Profile upload helper const HandleImageUpload = => { // we are referencing the file input const imageRef = useRef(); // Specify the default image const [defaultUserImage React Filemanager. Download Nulled Filedash – File Manager Dashboard. 1 - Added new Scrollable layout. x, Columns: 4+. FTP Access Upload files via FTP Need easier and faster way to upload and download. The default locale of the file manager is en (English). In our editor / file manager we should see a . It is distributed through NPM under the kendo-react-upload package. It was initially called Filer but was changed to Thunar due to a name clash. It can be used as a standalone app or as a middleware. I will try to make it clean and retro-compatible with the previous bridges/connectors. When it comes to both of these issues, React can help you provide a better user experience. Themes and Skinning JavaScript - jQuery, Angular, React, Vue React Data Grid. Build files will be created build. Select, Copy, Paste, and Delete. Python dictionary add, delete, update, exists keys with performance; java. react-dropzone is a React’s implementation of popular drag and drop library for file uploading. And in our opinion, the Webix library offers the best solution available on the market. Free bootstrap snippets, examples and resources built with html, css and js. Adding React File Manager for PDF Library In the previous section, we added the File Server Node API component from filemanager (by -OpusCapita) . So follow the below setups:- 1) Install the @react-native-community/checkbox package like below in your application 2) Link the module in your application 3) Import Free React Design System For Bootstrap 4 (reactstrap) 9,824 4. You can add spans to any grid element, fine-tune the table sizes, specify the columns’ auto width, and freeze one or more columns. Folder based file browser given a flat keyed list of objects, powered by React. 10. Client React connector for Google Drive API v2; Detailed documentation for each package is coming soon. You can fire up the project with dotnet run to see what the scaffold does for you. /assets/images/defaultUser. It is fully responsive, built with Bootstrap 4 Framework, HTML5, CSS3 and SCSS. The editor is the whole CodeSandbox application (file manager, code editor, dependency settings) and the preview is the result you see on the right. Extension for Visual Studio Code - Simple extensions for React, Redux and Graphql in JS/TS with ES7 syntax Another file format that uses the MD file extension is Moneydance Financial Data. import React, { useEffect, useRef, useState } from 'react'; // Specify camera icon to replace button text import camera from '. - Added Blog List, Blog Grid, Blog Details pages. Site Navigation and Layout. React Native This is an exact mirror of the React Native project, A lightweight and easy-to-use password manager Clonezilla. All the operating systems got a file manager to filter the required files. Create React App – How to Create and Deploy a React Application to Production. and we can drill down into various modules. 5. 0 To do so, right-click the libman. By default, Storybook's webpack configuration will allow you to: Import Images and other static files Semantic UI React provides React components while Semantic UI provides themes as CSS stylesheets. mov, . (Ex – Facebook, Twitter and Google. Then add the File Manager component as shown in below code example. Web. View . This is one of the admin tools that our customers manage their static files on shared host. Initially, the selectedFilestate is set to null The FileBrowser dialogs consist of a FileBrowser object, an object previewer/property manager and a file uploader tab. Let’s install Bootstrap and React. ” - [source] You can delete the files from My Media, My Documents or My Photos folders. Managing your React. Site Navigation and Layout. This can be done in one of two ways: Run bower install --save for each package (the --save flag adds the dependencies (name and version) to the bower. fThe file is called : "FirstReactApp. bs4 File Manager. A partition and disk Adds React debugging tools to the Chrome Developer Tools. This is a Sample React Plugin for Apache OODT File Manager. The FileBrowser dialogs consist of a FileBrowser object, an object previewer/property manager and a file uploader tab. Thunar is designed to start up faster and be more responsive than some other Linux file managers, such as Nautilus and Konqueror. rtl8761a_mp_chip_bt40_fw_asic_rom_patch_8812ae_new. This is an example file with default selections. new file manager windows 10 Executive Summary These course materials were originally designed for Google managers to help them transition from an individual contributor role to a manager role. jsx". dll and React. filebrowser provides a file managing interface within a specified directory and it can be used to upload, delete, preview, rename and edit your files. Files. json must be written in JSON. I will try to make it clean and retro-compatible with the previous bridges/connectors It's very important for me your collaboration on my development tasks and time. They are not part of the template and NOT included in the final purchase files. So in the above imports, the files would be CartTotal. In XCode, in the project navigator, right click Libraries Add Files to [your project's name] Go to node_modules react-native-file-manager and add the . LibraryManager. You can go for either an HTML5 drag & drop file uploader or use the traditional way. json file is saved. I want somebody who can redo what has been done and finish it. On the file manager for a domain you have a ‘public_html’ folder. dll (if using MVC 4) in your Web Application project Your first build always needs to be done using the build script ( dev-build. The FileBrowser provides the ability to browse directories and locate a file item. Downloading the file. cs ). ) Wrap long file names in the File Manager’s detail view Customize icons in the Folder Tree. com, lands you with the opportunity of working with a leading technology organization. v 2. You Install react-file-reader (A flexible ReactJS component for handling styled HTML file inputs. files[0]) } On saving, create-react-app will instantly refresh the browser. babelrc configuration file. Looking for the best react native app development companies? Here is the list of the top React native developers with reviews by ADA. Unfortunately it can be quite intimidating. if I add/remove files in File Manager, it will react dynamically on the front-side (so I don’t need to modify the shortcode or put a React is a popular open-source JavaScript library – many of you asked for an easier integration between Power BI and React web applications. Free bootstrap snippets, examples and resources tagged with file-manager, html, css and js. Themes and Skinning JavaScript - jQuery, Angular, React, Vue React Data Grid. Please help me to move forward with a donation by paypal :) The file manager component is used to browse, manage, and organize the files and folders in a file system through a web application. js. js file to a . Build) to the project, which will trigger a restore as part of project build. json file and choose “Enable Restore on Build”. Any FOSS lover is warmly welcomed A lot of people name React components with a capital letter in the file, to distinguish them from regular JavaScript files. To delete one or more files, 1. When viewing a module locally, the files are contained within module-name. Input A file input management component for React. WP Media Folder v5. Upload React website to subdomain Open File Manager Create new folder inside “public_html” Upload whole content of “build” folder into this new created folder. The “React JS Developer” role at one. How can I create a custom command for the Kendo UI File Manager? Creating a Custom Command. Restore on demand Library Manager will restore client-side libraries whenever the libman. It allows the creation of multiple users and each user can have its own directory. In this tutorial we are going to create a task manager application from scratch with react. Once a file item is selected, it (or its properties) is loaded in the previewer. The DevExtreme JavaScript FileManager component allows you to display and manage files and directories for different file systems. To make the functions work as expected, I transpile these into CommonJS format in addition to transpiling React JSX files. 부디 도움이 되길 바랄 뿐입니다. It's a command-line utility connected with the corresponding online repository of packages and is capable of package installation, version management, and dependency management. Step 9: Configuring AVD Manager. This time with Trillo File Manager is an application for Dropbox-like functionality on the top of the GCS cloud storage bucket. However, you don’t want to use the standard file input HTML element, instead use a styled link or button to show the file window. 3 - Binding of UI elements to JavaScript object models. 2. React is one of the best choices for building modern web applications. js file. Default configuration. This is a Sample React Plugin for Apache OODT File Manager. React has a slim API, a robust and evolving ecosystem and a great community. If nothing happens, download Xcode and try again. com and its affiliated web properties is provided "as is" without warranty of any kind. After downloading the installation file of it, double click on it and proceed with the installation. The KendoReact Upload component is part of the KendoReact library of React UI components. common. Say “EDIT MODE”. json React Component by Creating. You can delete the files from My Media, My Documents or My Photos folders. The new React component supports both JavaScript and TypeScript and will help you embed your analytics in a React web application. Our file caching system will have two main parts. Tailor fman to your needs with its powerful plugin system. React can handle a single button, a few pieces of an interface, or an app's entire user interface. js. Requirements Creating a File Upload Component with React. react-native-azure-blob-storage-manager. npm install react-files --save Usage Basic I don't think there is one, but it's such a strange question, React is used on the web as a frontend library, while a file manager runs on your local computer. Then use the Axios library to send the file request to the Laravel server and saves the image in the server. ReactOS is a free and open-source operating system for x86/x64 personal computers intended to be binary-compatible with computer programs and device drivers made for Windows Server 2003. Component { render() { return ( ); } } export default App; File Manager can be initialized using the tag. npm start To create a new build inside dist directory. Disclaimer: The information provided on DevExpress. Download Manager is a system service which optimizes the handling of long-running downloads in the background. The first is a React component, which will wrap around RNFetchBlob’s functionality and respond to changes in the Redux store. This method of deleting corrupted files requires you to close "Windows Explorer" through "Task Manager". json. A list of its key features is given below. The MD file stores transactions, budgets, stock information, bank accounts, and other related data for the Moneydance finance software. The second is a set of actions and reducers on the Redux store which deal specifically with file caching. apiOptions, apiRoot: `http://opuscapita-filemanager-demo-master. Integrate TinyMCE editor in Laravel with a File Manager / Image Upload Jquery PHP July 27, 2019 2,006 views Create Sortable, drag and drop multi-level list with jquery like wordpress menu page All Webix widgets and applications function well in the Angular or React environment. Today we will create File Manager App UI with animation using React Native. Sweet Alert in dark layout; Design Files Removed. Installation. Free bootstrap snippets, examples and resources tagged with file-manager, html, css and js. . dll. When a TypeScript script gets compiled there is an option to generate a declaration file (with the extension . You have high end protection; It also has a file manager that is easy to access. With React, you can create reusable components that are independent of each other. Use the Download button in the toolbar. The following table represents the default texts and messages of the file manager in en culture. This project based course will introduce you to all of the modern toolchain of a React developer in 2020. For example, users can upload images, videos, etc on Facebook, Instagram. Source + Demo. Data List; React DataTable Component Vue based front-end for File Manager Aug 01, 2018 1 min read. To associate your repository with the react-electron topic, visit your repo's landing page and select "manage topics. . 6. Benefits of Hosting React. This will add the LibraryManager NuGet package (Microsoft. Add events to precisely control file/folder operations (folder creation, file uploading, moving, deleting, etc. 6. Go through the following steps for creating React project to download file from server using React. A full list of the compatible frameworks and integration examples you can find on this page . Files and folders in the file system can be sorted in either ascending or descending order simply by npm install --save @opuscapita/react-filemanager @opuscapita/react-filemanager-connector-node-v1. If nothing happens, download GitHub Desktop and try again. import ReactFileReader from 'react-file-reader'; class Because the other files & folders above (some truncated) are usually part of a default react-native init installation, our focus would be on the src folder:. tsx. 5, npm 6. Drag & Drop your files in folders: Drag & Drop and image to move it into a folder, where you can find other files. To configure the AVD Manager click on the respective icon in the menu bar. These first have been selected by most active users and ranking has been given based on the most popular votes. Accessible , tested with AT software like VoiceOver and JAWS, navigable by Keyboard . changing a . target. Install from NPM and include it in your own React build process (using Browserify, Webpack, etc). The project is about uploading a users products/services. The name npm (Node Package Manager) stems from when npm first was created as a package manager for Node. If multiple packages depend on a package - jQuery for example - Bower will download jQuery just once. React. View . Communicating react with asp. holyidiot updated Vuexy - Vuejs, React, HTML & Laravel Admin Dashboard Template with a new update entry: Update [6. 0] – 2020-11-28 Latest Update [6. ts) that functions as an interface to the components in the compiled JavaScript. Added 2021-01-09 file-manager,file-browser spofly Desktop app to find lyrics of currently playing song on spotify. You have two options for creating a file uploader. ReactOS will only be compatible with computers that are compatible with Windows 2003 or XP. ej2-react-filemanager. Basic usage. Any FOSS lover is warmly welcomed React Native ; Bootstrap file-manager examples. Bower provides hooks to facilitate using packages in your tools and workflows. jsx" Select Web => JSX File, and enter file name "FirstReactApp. The FileManager provides an inbuilt Search functionality, allowing you to find the specific file in the currently selected folder. React File Manager: A Lightweight & Customizable Component File upload and download. File Browser Front-end. Paper Kit ReactOS is a free, opensource reimplementation of windows Related: How to Copy and Paste Text, Files and Folders in Linux Terminal. Developer Express Inc disclaims all warranties, either express or implied, including the warranties of merchantability and fitness for a particular purpose. Removed complementary design files from the package [5. Uploading Files using HTML5 Uploader. TagSpaces features basic file management operations, so it can be used as simple file manager. To associate your repository with the react-electron topic, visit your repo's landing page and select "manage topics. dhtmlxGrid contains rich API functionality. This is an unparalleled array of features, design elements and reusable components Introduction to Asus file manager. To do so, right-click the libman. /assets/images/camera. Try All UI Components for Free All UI components for React are part of the dhtmlxSuite library. Changing the Webpack config. Add multiple URL to pocket at a time. Create better React apps faster and add data visualizations with the world's fastest, virtualized, real-time React data grid and streaming financial and business charts. File uploading means a user from a client machine wants to upload files to the server. net core on remote server (httpdocs folder) does not work RSS 0 replies Last post 2 hours, 59 minutes ago by fiazahmed An electron based file manager. 10 – WordPress File Manager Using the default WordPress media manager also means that the plugin will be very compatible with all the other plugins you use. It’s not a visual file manager, but it gives a set of functions to easily handle media/files in your Laravel app. html’ file to run on start up. Once a file item is selected, it (or its properties) is loaded in the previewer. Read the full article at: http://bgwebagency. The presence of these handlers enables the buttons and/or the drag & drop responsiveness. (eg. However, newer versions of the program use . The FileManager uses file system providers to access file systems. fastlane/ This folder, as you might React doesn’t have opinions on how you put files into folders. files[0]) } On saving, create-react-app will instantly refresh the browser. No action needed. pdf file to use a . Firebase issue during npm install [5. The standard tool for this task is Babel. The create-react-app utility configures tools such as Babel and webpack for the client-side React application. Build) to the project, which will trigger a restore as part of project build. In this section, we are going to add the Client React component from OpusCapita for navigating the folders and listing the files in our PDF library. nyc_output and coverage folder containing our instrumentation detail. . In terms of frontend frameworks, this React admin dashboard is powered by Material-UI, which is the most popular material-based UI components framework available today. - Added File Manager Page. MONEYDANCE files instead. Blog Post. Page 16. lang. A file input (dropzone) management component for React. Videos: Manage member level settings & Videos created by Users. storybook/main. ) Channels: Manage member level settings & Channel created by Users. It enables the user to perform common file operations such as accessing, editing, uploading, downloading, and sorting files and folders. To upload a file with React and Laravel, create a React file component and backend in Laravel. 2. React Chart. The following sample is extracted from Android 4 ICS emulator image. Similarly, every smartphone has a file manager to view, edit, and create any text files, delete, sort, or rename, copy, and cut whenever required. All npm packages are defined in files called package. ). The official front-end framework for building experiences that fit seamlessly into Microsoft 365. Create shortcuts for files: Hold SHIFT and move a file with drag & drop to another folder in order to create a shortcut Bootstrap snippets. 1. js file. To select a specific file, you need to use the number assigned to it. See bundler defaults for the full list. Bower is optimized for the front-end. 15. React. This allows teams to set conventions that work best for them, and to adopt React in any way they would like to. It also supports uploading a file by dragging it from Windows Explorer to FileManager control. Cezerin is open-source ecommerce platform. Tailor your React grid component according to your needs. dll GitHub - networknt/react-file-manager: A react remote file manager with Light Framework as back end file system. Social Sites Integration: With one click, you can login to your site using Social Sites. Using the arrow keys, move over the desired file or folder and press Space on the keyboard. 9 all file uploads, including those initiated by the File Browser plugin, expect a JSON response (like this one ). In addition to building skills, this curriculum incorporates introspection, perspective shifting, and awareness building. These two parts are very decoupled and only communicate using postMessage. Plugin has two parts: Front-end & Back-end. Note that your changes would be temporary and will not persist between re-runs of your package manager. Other Feature Module in React template: Voice and Video Call File Manager Contacts and Email Departments and Designations Timesheet and Overtime Kanban Board Payroll, Payslip and Payrun Company Policies Performance, Goal Tracking, Training and Promotion Resignation and Termination Faq and Knowledgebase Profile Settings, Profile and Edit Profile 🎈 React Material-UI. August 08, 2018. Store the file in state, and only upload when a user clicks the upload button. Overview. This package help you to upload file and assests from react native project to your azure blob storage service. If you want to download large files/streaming you can use Android Download Manager. React components can be installed via yarn or npm: After install, import the minified CSS file in your app's entry file: File manager built with the help of Suite components: Layout, Grid, DataView, Toolbar, etc. js doesn’t have to be hard and with these few steps, you can do it yourself. js'; class App extends React. Use npm to install the Expo CLI command line utility from the Windows Command Prompt, PowerShell, Windows Terminal, or the integrated terminal in VS Code (View > Integrated Terminal). Clear the cache from admin panel. js , or Total. How you use packages is up to you. html file should reveal our code coverage in a human readable and hopefully revealing way. We will cd into react project react-file-upload – cd react-file-upload Now will install dependencies – npm install bootstrap npm install react-toastify npm install axios The bootstrap help to create ui based on bootstrap 4, react-toastify is use to display beautiful notification into react app and axios for HTTP client. Save time by quickly jumping to directories. CKEditor 4 can be easily integrated with an external file manager (file browser/uploader) thanks to the File Browser plugin which by default is included in the Standard and Full presets. A very smart filemanager to manage your files in the browser developed in AngularJS following Material Design styles by Jonas Sciangula Street. These files will always be rendered/loaded to the page when an instance of the module is on the page (Module instances are the individual rendered modules on the page). FileManager Conclusion To make a file downloadable from your website, start by creating a folder on your server for both your website's HTML page and the file you want to share. Select the file in the manager. It is often used for developing Web Applications or Mobile Apps. mp3, . Edit: But As a web app. 2. You can add a custom thumbnail and text description to every file or folder. That said there are a few common approaches popular in the ecosystem you may want to consider. changing a . Drag and Drop Support in React FileManager component 23 Feb 2021 / 1 minute to read The file manager allows files or folders to be moved from one folder to another by using the allowDragAndDrop property. Once you make the folder, you can find it by using your Control Panel's file manager or the file browser in your FTP program. In XCode, in the project navigator, select your project. files[0]holds the actual file and its details. react-file-manager. React Native ; Bootstrap file-manager examples. Download the App Center SDK for React Native frameworks provided as a zip file and unzip it. React, Node v12. A dual-pane file manager for Mac, Windows and Linux. Restore on demand Library Manager will restore client-side libraries whenever the libman. html, the rest of the website build in React shows nothing (white page). Set Api path for React. To connect the component with the file system items, assign the Remote File System Provider to the fileSystemProvider property. React File Manger Multi-column File Manager based on react-beautiful-dnd. React JS Developers one. js. Here are some of the best places to find up-to-date information on React and TypeScript: React TypeScript Cheatsheets React is a JavaScript library that aims to simplify development of visual interfaces. svg'; // replace it with your path // Specify your default image import defaultUser from '. json. File Manager. css'; import 'devextreme/dist/css/dx. js under dist/ directory. /. The FileBrowser provides the ability to browse directories and locate a file item. Chocolatey is software management automation for Windows that wraps installers, executables, zips, and scripts into compiled packages. React Chart. npm run build. files[0]holds the actual file and its details. This one is a little different. All of the files shared are under GPL License. Like a photo, pdf or any other file type. 3] - 2020-04-04 VueJS + Laravel, HTML + Laravel Updated. in/building-a-full-st Related Posts: How to download file from server using Angular; Prerequisites. Cronus File Manager Live Demo. Files files will be hosted on the server on a cloud service. js under dist/ directory. Allows creating a Progressive Web Apps built with React and Node. A flexible and beautiful Select Input control for ReactJS with multiselect, autocomplete and ajax support. You can add support for other types by adding an assetExts resolver option in your Metro Bower keeps track of these packages in a manifest file, bower. Please note that bluehost doesn’t upload folder and its content. Most common file types are supported including . Scheduler. Go to the My Media/My Documents/My Photos folder. If you don’t have that file already, you just create a blank file, and put that content into it. js is an open-source JavaScript library that is used for building user interfaces specifically for single-page applications. The KendoReact Upload helps users send files from their file systems to dedicated server handlers which are configured to receive them. Basic usage. Mobile applications definitely offer a greater value to businesses than their mobile website • Fixed Spell checker not working and missing Image Advanced Edit button in Node JS SDK • Fixed Unable to load any images or files Python Flask SDK • Fixed Upload Video not working in Rail SDk • Fixed On opening an uploaded file throws "HTTP status 404-Not Found" in Java SDK • Fixed Unable to upload images in Java SDK • Fixed On opening an uploaded file throws "Template is missing The require syntax described above can be used to statically include audio, video or document files in your project as well. If nothing happens, download GitHub Desktop and try again. xml :This file contain list . com and its affiliated web properties is provided "as is" without warranty of any kind. As of Kendo UI R1 2020 SP1 the kendo. Go to the My Media/My Documents/My Photos folder. Click on the Delete button. Material Dashboard React Nodejs . ) Simple Example. 2] - 2020-02-18 React Aaded. react-dom@^16. npm install -g expo-cli Use Expo to create a React Native app that runs on iOS, Android, and web. For the former, there is a library called react-dropzone that is built with React. File Operations. This happened right after updating the code when I tried to upload some . Pass the endpointUrl to the remote file system provider object to specify the Url at which the component can access the file system items. First we need to install the dependencies for React. ui. Note: In the documentation, the package babel-plugin-styled-components is specified, as well as a . This project provides a web file manager interface, allowing you to create your own backend connector following the connector API. 3 - Upgraded React version to 17. However, it is not designed to work with SSR. target. Mvc4. There’s nothing more to add, just check out our demo to get a clear idea of what you can do with it. It’s used for handling the view layer for web and mobile apps. jsx) by right clicking on container folder script => react folder select a file from new items dialog popup and click on Add button. /data. This is where Babel macros come in. 07 August 2019. This prod Nowadays, Node Package Manager (npm) is one of the most demanded gadgets in the web developer tool belt. You can open the Task Manager by 2 options. The value should be an async function that receives a webpack config and eventually returns a webpack config. js on cPanel. Like a photo, pdf or any other file type. Along the way, we will build a massive e-commerce application similar to Shopify using React, Redux, React Hooks, React Router, GraphQL, Context API, Firebase, Redux-Saga, Stripe + more. js on cPanel. bat ) as this generates a few files required by the build (such as SharedAssemblyVersionInfo. With this in place, feel free to open the solution file in Visual Studio or VS Code. Free Web File Thunar is developed by Benedikt Meurer, and was originally intended to replace XFFM, Xfce's previous file manager. React Scheduler Storybook is an open source tool for developing UI components in isolation for React, Vue, and Angular. Ignite UI for React also includes the most complete Microsoft Excel solution and 60+ chart types with interactive panning and zooming, touch support and much more. An online file manager which can be used on its own, or as a plugin for a rich-text editor such as CKeditor, TinyMCE or FCKeditor. png extension) Let’s take a quick look at how to manage those breaking user interactions: to the . Option 1: Type "task" in the search box beside the Start menu, and press Enter when you see the "Task Manager" app. /. export const fileItems = [{ 'name': 'Documents', 'isDirectory': true, 'category': 'Work', 'items': [{ 'name': 'Projects', 'isDirectory': true, 'category': 'Work The Custom File System Provider allows you to implement custom APIs to handle file operations (add, delete, rename, etc. Initially, the selectedFilestate is set to null Next time you’re looking for a file, it’s just a click away in the file manager. This will add the LibraryManager NuGet package (Microsoft. . The Edit screen with option to select one or more files is displayed. js) as shown below. Webix suggests a ready-made solution, which is JS File manager, that can be built into any web application. Fileside Modern, tiling file manager with unlimited panes. I have a demo on Laravel + React. React, Redux, Material UI, Nodejs, ExpressJs . We use Gatsby with TypeScript for this website, so that can also be a useful reference implementation. module folders. Work fast with our official CLI. To select a file or folder: 1. log(event. Store the file in state, and only upload when a user clicks the upload button. wav, . You can then use the Dropzone component to render the HTML5 Drag What we would like to see from a project manager is the following: - A candidate that can manage: 1 - Experience with React context api . 90/5. Declaration files. It come with unlimited customized email with your domain. Looking for the best react native app development companies? Here is the list of the top React native developers with reviews by ADA. To select a specific file, you need to use the number assigned to it. Angle - Responsive Bootstrap Admin Template. Use it as a child component of you application. As with any programming problem, there are many ways to achieve this outcome. Run npm install and npm start after that. For initialising file manager you have to install and run both of them from terminal with commands . 3. 5. react-files. /. A predictable state container for JavaScript apps. xcodeproj file. JSX Now, we need to create a first component to create a file (. A simple file manager built with react. Client implementation is an npm package which can be embed into your application. Grouping by features or routes One common way to structure projects is to locate CSS, JS, and tests together inside folders grouped by feature or route. yarn add react yarn add react-dom yarn add --dev parcel-bundler. 11, React 16/17. The File Manager is a graphical user interface component used to manage the file system. mp4, . Delete a file. Web. expo-file-system ( docs) expo-media-library ( docs) After you’ve done that we can proceed. Create shortcuts for files: Hold SHIFT and move a file with drag & drop to another folder in order to create a shortcut There are several possible ways of using Webix with React: using a Webix widget in a React app; creating a custom Webix+React component; using a Webix widget with Redux; How to Start. Option 1: Package Manager. This article explains a simple way to implement the approach to upload a single file with React. Software Package Manager. View demo Download Source. Say “MORE OPTIONS” 3. Free . You can assign custom color to every folder and tag, which makes the visual search an easy step. - Fixed minor bugs. net/` // Or you React File Manager Usage (iOS) First you need to install react-native-file-manager: npm install react-native-file-manager --save. " File Manager. Step 8: Configuring AVD Manager. On the backend, we are going to use Laravel’s Storage API to store images. Now, you can start adding Essential JS 2 File Manager component to the application. 3. All basic file handling mechanisms like upload, download, read, edit, delete, search, and sort can be performed to manage and organize the files and folder in a file system. ly/3d8cXTx To learn more about the react-native visit: The FileManager UI component can work with a file system located on the server. Angular React Vue jQuery PeaZip is a free archiver tool. config. com is looking for React JS Developers for our team in Delhi/NCR (India) Employment: Permanent Employment Place of Work: Delhi/NCR (India) CTC: Best in the industry Role. Complete file and folder manager: Create, rename, move and delete a folder. Reference React. 0 / scheduler@^0. Use the fileSystemProvider property to configure the component's file system provider. Video-React is a web video player built from the ground up for an HTML5 world using React library. Let’s begin with the Redux side of things: The Redux Code Unlike the other frameworks covered in this module, React does not enforce strict rules around code conventions or file organization. Free Frontend Preset For Nodejs . That's when we got the idea to create an orthodox WEB file manager, working on the server's site, which would be able to copy between different sources with server speed and would offer: file and directory search, a disk usage analyzer (an analogue of ncdu), simple file uploading and a lot of other great stuff. In traditional HTML sites, the file upload form forces a page refresh, which might be confusing to users. Chocolatey integrates w/SCCM, Puppet, Chef, etc. The React File Manager component allows for the easy uploading and downloading of files in a Sorting. KFM – Kae’s File Manager. Webix File Manager is a ready-made SPA. " For example, I prepare a page “Commercial files” where I will put a shortcode corresponding to the folder of files uploaded in File Manager or Google Drive. React was first created by Jordan Walke, a software engineer working for Facebook. Web The JavaScript Client Library for Azure Storage enables many web development scenarios using storage services like Blob, Table, Queue, and File, and is compatible with modern browsers. npm install npm run start Design. thumbnail support for image files; built-in media player; text editor; many other features. I want to do a very simple file explorer in react that look like the one of Files for google. Multi-Selection. js, and Mongo. React Shopping Cart. More Template Epic React - HR Management Admin Template is High Resolution: Yes, Compatible Browsers: Firefox, Safari, Opera, Chrome, Edge, Compatible With: ReactJS, Bootstrap 4. These first have been selected by most active users and ranking has been given based on the most popular votes. So follow the below setups:- 1) Install the @react-native-community/checkbox package like below in your application 2) Link the module in your application 3) Import Get code examples like "usenavigate react" instantly right from your google search results with the Grepper Chrome Extension. 6. Disclaimer: The information provided on DevExpress. 0/v14. css'; import FileManager from 'devextreme-react/file-manager'; import { fileItems } from '. Since CKEditor 4. In the process the compiler strips away all function and method bodies and preserves only the signatures of the types that are exported. You have to manually create sub-folder, then upload files into that folder. React Filemanager Hello ex angular-filemanager user, this is the new version in React. The application provides an unified, natively portable, cross-platform file manager and archive manager GUI for many Open Source technologies like 7-Zip, FreeArc, PAQ, UPX. But first, here are the benefits of hosting your React. The ASP. It is developed by laborasyon on ThemeForest. In this tutorial, we will upload an image from the react js component. 80/5. React is an open-source JavaScript library developed by Facebook used for creating web frontend and UI components. 14. File Manager and Core Data: Used to save photo, video, audio, and pdf data to the ios device url sessions: Used to communicated with the server to upload the data to the Utah State Geographical Cuba admin is super flexible, powerful, clean & modern responsive bootstrap 5 admin template with unlimited possibilities. Or if you have the optional Yarn package manager installed. React & JavaScript articles. The File Manager component supports multiple selections of files and folders in a file system. Storybook - GitHub Pages angular-filemanager. It is worth noting the beautiful design, and a ready-made set of icons, which are included in the delivery. 10. Work fast with our official CLI. Beside Material-UI, we also integrated, with the same design style, over 80 React widgets and plugins. Click on the Next button you will see a System React Fixed. Developer Express Inc disclaims all warranties, either express or implied, including the warranties of merchantability and fitness for a particular purpose. NullPointerException; TypeError: string indices must be integers – Python; valueerror: setting an array element with a sequence – Python; TypeError: a bytes-like object is required, not ‘str’ – Python Drop files, select on filesystem, copy and paste files, or add files using the API. To start the app server it will display live changes (optional) 4. 그럼 스타뜨! 배경 DCE내에서 파일 업로드 및 관리를 할수 있는 GUI 화면이 필요했다. React Scheduler Disclaimer: The information provided on DevExpress. The issue with this is that, because we’re using create-react-app, we can’t configure a lot of things unless we eject. This package support multiple files selection, cloud storage integration. Creating a file upload component is a common task in web development. Developer Express Inc disclaims all warranties, either express or implied, including the warranties of merchantability and fitness for a particular purpose. At least two fields must be present in the definition file: name and version. 0] – 2020-11-28 HTML, HTML + Laravel ADDED All-new design based on Ul/UX principles New Bordered & Dark layout New eCommerce Dashboard Invoice Bamburgh React Admin Dashboard with Reactstrap PRO is built entirely on React and uses the popular starter kit Create React App from Facebook. Now my APP only show that green circle button located in index. e. Just be sure to follow the installation instructions for “bare” or plain react-native apps. json file. Build files will be created build. First, we install dependencies using npx then download the laravel project. 4. Download Epic React – HR Management Admin Template nulled from the below download links and if the item satisfy you then buy it from the developer puffintheme for commercial use. - Minor fixes of RTL SCSS. npm install --save react npm install --save react-dom npm install --save-dev parcel-bundler. js - TS docs; Gatsby - TS Docs; All of these are great starting points. Finally, what all this was leading up to, opening that index. I guess it's technically possible to write a file manager in Node, use React for the UI, and package it as a desktop app with Electron, but I would still not call that "React based" (and C. js in your Greg Fodor - Engineering Manager Mozilla The development team involved have been very impressed by the React Admin framework and it has been capable of handling the complex challenges we have had for it thusfar. It uses React framework and supports connectors to different file storages. Drag & Drop your files in folders: Drag & Drop and image to move it into a folder, where you can find other files. File Manager: Admin can import/export & upload new files. packages. Simple event handlers are also provided as props to the browser, which allow it to respond to actions on the files. Thus you will get an example of integration usage. onChangeHandler=event=>{ console. . It has a large UI collection. dll. onChangeHandler=event=>{ console. import React from 'react'; import ReactDOM from 'react-dom'; import { FileManager, FileNavigator } from '@opuscapita/react-filemanager'; import connectorNodeV1 from '@opuscapita/react-filemanager-connector-node-v1'; const apiOptions = { connectorNodeV1. json file is saved. API-first CMS. json file and choose “Enable Restore on Build”. FileManager also performs operations like creating a new folder, moving files, and searching. This control is part of the Telerik UI for ASP. Created from revision f160547f47 on 12/4/2020. All APIs that implement access to Azure Blob Storage on the client are stored in the azure-file-system. banzay/friends-app-redux Second take on friends app. 4 - Creating RESTful services with Package Manager stores application information in three files, located in /data/system. Is the Excel Viewer widget compatible with the Webix community (opensource) edition? PHP & MySQL Projects for $2 - $10. rtl8761a_mp_chip_bt40_fw_asic_rom_patch_8192eu_new. d. Select the file to upload from the file selector dialog box; Downloading a file. pdf. NET Core FileManager lets your users browse through directories and files, akin to file managers like Windows Explorer, and manage file storage within their web applications. spatie/laravel-medialibrary Released: August 2015 Installs: 178 000 Last update: May 2017 (1 day ago). Note the command dotnet new react; this is the template I’m using for this React project. Your domain will look in this ‘public_html’ folder for a top ‘index. You'll see a folder named AppCenterReactNativeShared which contains a single framework for the required React Native iOS bridge. Be it a web-based gaming experience where you store state information in the Table service, uploading photos to a Blob account from a Mobile app, or an entire CodeSandbox at its core consists of two parts: the editor and the preview. Predefined connectors are: Client React connector for Server Node API v1 Localization in React FileManager component The file manager can be localized to any culture by defining the texts and messages of the file manager in the corresponding culture. babelrc file present in the application root folder. jpeg extension) Uploading an image where the file extension has been intentionally changed and Cloudinary could process it, but the DOM could not render the file (eg. Web based File Manager Manage files online From within the free control panel, an easy to use File Manager helps you to upload files, download files or even edit HTML, PHP or other programming language files. 2. The content of package. Create a new file called manager. All basic file operations like creating a new folder, uploading and downloading of files in the file system, and deleting and renaming of existing files and folders are available in the file manager component. Used technologies. askwon/Filet-Manager Web-based file transfer client written in React, Redux, and Go; ayxos/react-cellar Typescript, MongoDb, Webpack, EC6, Typings, Redux Wine-Cellar; azu/read-all-later [Electron] Read All Later is a client for Pocket. - Added New Auth pages. To run the service, create an Amazon S3 account and a S3 bucket and then register your amazon S3 client account details like bucketName, awsAccessKeyId, awsSecretKeyId and awsRegion details in RegisterAmazonS3 method to perform the file operations. 9. 5. Angle is an admin template based on Bootstrap and multiple frameworks. Personalize your React grid with flexible API. Organizing your blog media files with the Real Media Library plugin is as easy as dragging and dropping them into folders. JavaScript File Manager or in other words File Explorer is a web widget, part of the UI framework for managing files. Developed at Facebook and released to the world in 2013, it drives some of the most widely used apps, powering Facebook and Instagram among countless other applications. light. . Here native file viewer means we are not going to view the file in our application instead we will pick the file from file picker and will pass the file URL to FileViewer component provided by react-native-file-viewer, this component will trigger the native iOS/Android file viewer to open the file. Say “MORE OPTIONS” 3. ). To delete one or more files, 1. To download a remote file’s content to a local file on the device, here’s the code: Hi Dev, In this blog, I will show you how to install file manager package in laravel application. Install Step 1 npm i react-native-azure-blob-storage-manager --save Step 2 Dependencies npm install --save react-native-background-upload iOS cd ios pod install Manual Installation Installation. Node. Use Git or checkout with SVN using the web URL. Page 16. Documentation. We are going to use react-dropzone to build an image uploader. Hello ex angular-filemanager user, this is the new version in React. react file manager diff --git a/src/main/resources/sdtocode/doc/Apache OODT File Manager/React file manager.txt.xml.xls b/src/main/resources/sdtocode/doc/Apache OODT File Manager/React file manager.txt.xml.xls deleted file mode 100644 index b162ca54b5175f545e6d964c9ad0ef56ec5c5dc9..0000000000000000000000000000000000000000 Binary files a/src/main/resources/sdtocode/doc/Apache OODT File Manager/React file manager.txt.xml.xls and /dev/null differ diff --git "a/src/main/resources/sdtocode/doc/Apache OODT File Manager/cas-filemgr \342\200\223 CAS File Manager Developer Guide-relation.txt" "b/src/main/resources/sdtocode/doc/Apache OODT File Manager/cas-filemgr \342\200\223 CAS File Manager Developer Guide-relation.txt" deleted file mode 100644 index adfa85b220fdb253bc01ab03e47341934f482e06..0000000000000000000000000000000000000000 --- "a/src/main/resources/sdtocode/doc/Apache OODT File Manager/cas-filemgr \342\200\223 CAS File Manager Developer Guide-relation.txt" +++ /dev/null @@ -1,221 +0,0 @@ -Introduction This&Catalog&依赖 -its&points& -guide&tailorable extension point&依赖 -guide&File Manager architecture and interface&依赖 -our&Guides& -remainder§ion&依赖 -remainder&guide&AGGREGATION -we&architecture&依赖 -we&constituent component&依赖 -its&components& -we&architecture&依赖 -we&constituent component&依赖 -architecture&File Manager&AGGREGATION -Client and Server&major component&依赖 -Client and Server&File Manager&依赖 -major component&File Manager&AGGREGATION -relationship&diagram&依赖 -Catalog&metada&依赖 -File Manager Server&Repository&依赖 -Catalog&Validation Layer&依赖 -products&location& -File Manager Server&' location&依赖 -Repository&product&依赖 -File Manager Server&archive as&依赖 -Transfer&Transfer&依赖 -Transfer&Transfer&依赖 -Transfer&data product&AGGREGATION -Transfer&Transfer&依赖 -domain&Transfer&AGGREGATION -Transfer&Transfer&依赖 -Transfer&Transfer&依赖 -Transfer&Transfer&依赖 -collection&one or more file&AGGREGATION -their&Metadata& -A map&key&AGGREGATION -> multiple value&descriptive information&AGGREGATION -See&Metadata&依赖 -See&more information&依赖 -its&location& -File Manager&what type&依赖 -File Manager&file urus generation scheme&依赖 -what type&file urus generation scheme&AGGREGATION -description&Product&AGGREGATION -element&form&依赖 -element&associated definition&依赖 -form&associated definition&AGGREGATION -element&additional metada&依赖 -A URI generation scheme&location&依赖 -A URI generation scheme&built&依赖 -A URI generation scheme&archive (&依赖 -Product&1 or more reference&依赖 -member&single Product Type&AGGREGATION -Product&single Product Type&依赖 -Product&mapping&依赖 -Product&Product Type&依赖 -mapping&Product Type&AGGREGATION -Product Type&associated Versioner&依赖 -relationship&below figure&依赖 -new&key capability&AGGREGATION -Easy management&different type&AGGREGATION -different type&Products&AGGREGATION -their&information& -their&ID& -Management&Product Types&AGGREGATION -Management&new type&依赖 -their&name& -Management&new type&依赖 -different kind&back end catalog&AGGREGATION -Catalog extension point&extension point&GENERALIZATION -Catalog extension point&product instance metada and file location information&依赖 -different type&back end data store&AGGREGATION -implementation&end database&依赖 -implementation&JDBC&依赖 -implementation&end database&依赖 -implementation&JDBC&依赖 -implementation&Catalog interface&AGGREGATION -Management&Product instance information&AGGREGATION -Management&include add , delete and update product instance information&依赖 -Management&file location&依赖 -It&Metadata and reference&依赖 -Manager&point& -different type&back end store&AGGREGATION -management&Element policy information&AGGREGATION -Element policy&instance&依赖 -Element policy&XML file&依赖 -Data Transfer&Transfer&GENERALIZATION -File Manager&different Data Transfer protocol&依赖 -Versioner extension point&different File Repository layout&依赖 -Versioner extension point&extension point&GENERALIZATION -Flat product&singular file&依赖 -collection&singular file&AGGREGATION -Products&directory&依赖 -collection&directory&AGGREGATION -File Manager&popular client-server paradigm&依赖 -File Manager&XML-RPC&依赖 -its&interface& -File Manager&File Manager client and server&依赖 -File Manager&main external interface&依赖 -little brother&SOAP&AGGREGATION -File Manager web interface&RSS-based syndication&依赖 -RSS-based syndication&Product feed&AGGREGATION -datum&status tracking&依赖 -RSS-feed&transfer&AGGREGATION -File Manager¤t Product and File transfer&依赖 -Extension Points We&file manager make use&依赖 -file manager make use&factory method pattern&AGGREGATION -interface&many implementation&依赖 -extension point&File Manager&依赖 -different implementation&interface&AGGREGATION -it&software component configuration&依赖 -it&different implementation&依赖 -it&interface&实现 -File Manager extension point&implementation&依赖 -File Manager extension point&two interface&实现 -implementation&two interface&AGGREGATION -File Manager extension point&extension factory&实现 -File Manager load&factory class&依赖 -File Manager load&factory class&依赖 -File Manager load&run-time&依赖 -File Manager load&run-time&依赖 -File Manager&example&依赖 -File Manager&database-based Catalog&依赖 -it&Lucene-based Catalog&依赖 -it&many different type&实现 -The Data Transfer extension point&Product&依赖 -The Data Transfer extension point&movement&依赖 -The Data Transfer extension point&archive&依赖 -movement&Product&AGGREGATION -Different protocol&local ( disk-based ) copy&依赖 -Different protocol&local ( disk-based ) copy&依赖 -extension point&Product Type&依赖 -extension point&element&依赖 -different URI generation scheme&file&依赖 -different URI generation scheme&final resting location&依赖 -final resting location&file&AGGREGATION -different URI generation scheme&particular Product&依赖 -definition&different URI generation scheme&AGGREGATION -Catalog Data Source&base catalog&依赖 -implementation&Catalog extension point interface&AGGREGATION -implementation&a jdbc accessible database backend&依赖 -lucene base catalog&lucene base catalog&依赖 -implementation&catalog extension point interface&AGGREGATION -implementation&catalog extension point interface&依赖 -lucene base catalog&Lucene free text index system&依赖 -implementation&catalog extension point interface&依赖 -implementation&Data Transfer interface&AGGREGATION -Apache&commons-io& -implementation&locally accessible network file system ( nfs ) disk&依赖 -implementation&XML-RPC File Manager client&依赖 -XML-RPC File Manager client&File Manager client&GENERALIZATION -implementation&XML-RPC File Manager client&依赖 -implementation&datum transfer interface&AGGREGATION -InPlace Data Transfer .&product&依赖 -implementation&product type policy information&依赖 -implementation&product type policy information&依赖 -implementation&repository manager extension point&AGGREGATION -implementation&JDBC accessible database&依赖 -implementation&JDBC accessible database&依赖 -XML file&file&GENERALIZATION -implementation&JDBC accessible database&依赖 -implementation&JDBC accessible database&依赖 -implementation&element policy information&依赖 -implementation&element policy information&依赖 -implementation&validation layer extension point&AGGREGATION -Validation Layer extension point&2 XML file&依赖 -implementation&) xml-rpc&依赖 -Validation Layer extension point&Element policy information&依赖 -implementation&( file manager client&依赖 -implementation&) xml-rpc&依赖 -implementation&Validation Layer extension point&AGGREGATION -implementation&( file manager client&依赖 -File Manager&transportation medium&依赖 -implementation&File Manager&依赖 -File Manager&use xml-rpc&依赖 -implementation&File Manager&依赖 -File Manager&use xml-rpc&依赖 -implementation&File Manager&依赖 -File Manager&transportation medium&依赖 -implementation&external server interface&AGGREGATION -XML-RPC&File Manager client&依赖 -implementation&client interface&AGGREGATION -implementation&XML-RPC File Manager server&依赖 -XML-RPC File Manager server&transportation medium&依赖 -implementation&XML-RPC File Manager server&依赖 -XML-RPC File Manager server&XML-RPC&依赖 -implementation&XML-RPC File Manager server&依赖 -several&above capability&AGGREGATION -we&that&依赖 -manager ingest use case red number&step&依赖 -manager ingest use case red number&step&依赖 -sequence&step&AGGREGATION -series&interaction&AGGREGATION -manager ingest use case red number&sequence&依赖 -manager ingest use case red number&sequence&依赖 -ingest operation&ingest&依赖 -File Manager client&Step 1&依赖 -ingest operation&a particular product&依赖 -File Manager client&ingest operation&依赖 -ingest operation&Metadata and References&依赖 -server&point& -System Interface&information&依赖 -it&path& -System Interface&made&依赖 -System Interface&Product Type policy&依赖 -Metadata&Catalog extension point&依赖 -System Interface&file reference&依赖 -Catalog extension point&catalog process&依赖 -Catalog extension point&Validation Layer&依赖 -its&Type& -first step&’s associated versioner&依赖 -Product&Versioner& -first step&’s associated versioner&依赖 -Data Transfer&client or server end&依赖 -aim&document&AGGREGATION -document&architecture&依赖 -document&constituent components , object model and key capability&依赖 -Manager&architecture& -its&model& -current implementation&extension point&AGGREGATION -overview¤t implementation&AGGREGATION -Manager&points& diff --git "a/src/main/resources/sdtocode/doc/Apache OODT File Manager/cas-filemgr \342\200\223 CAS File Manager Developer Guide.txt" "b/src/main/resources/sdtocode/doc/Apache OODT File Manager/cas-filemgr \342\200\223 CAS File Manager Developer Guide.txt" deleted file mode 100644 index fbb01d68e919c4da23d6cedfdae91f79d032c64b..0000000000000000000000000000000000000000 --- "a/src/main/resources/sdtocode/doc/Apache OODT File Manager/cas-filemgr \342\200\223 CAS File Manager Developer Guide.txt" +++ /dev/null @@ -1,100 +0,0 @@ -Introduction -This is the developer guide for the Apache OODT Catalog and Archive Service (CAS) File Manager component, or File Manager for short. Primarily, this guide will explain the File Manager architecture and interfaces, including its tailorable extension points. For information on installation, configuration, and examples, please see our User Guides. - -The remainder of this guide is separated into the following sections: - -Project Description -Architecture -Extension Points -Current Extension Point Implementations -Project Description -The File Manager component is responsible for tracking, ingesting and moving file data and metadata between a client system and a server system. The File Manager is an extensible software component that provides an XML-RPC external interface, and a fully tailorable Java-based API for file management. - -Architecture -In this section, we will describe the architecture of the File Manager, including its constituent components, object model, and key capabilities. - -Components -The major components of the File Manager are the Client and Server, the Repository Manager, the Catalog, the Validation Layer, the Versioner, and the Transferer. The relationship between all of these components are shown in the diagram below: - -File Manager Architecture - -The File Manager Server contains both a Repository that manages products (and the products' location in the archive as specified by Versioner), and a Catalog that validates metadata via the Validation Layer. Transfer of data products from the Client to the Server is the domain of the Transfer and can be initiated at either the Client or the Server. - -Object Model -The critical objects managed by the File Manager include: - -Products - Collections of one or more files, and their associated Metadata. -Metadata - A map of key->multiple values of descriptive information about a Product. See CAS-Metadata for more information on Metadata. -Reference - A pointer to a Product file's (or files') original location, and to its final resting location within the archive constructed by the File Manager. -Product Type - Descriptive information about a Product that includes what type of file URI generation scheme to use, the root repository location for a particular Product, and a description of the Product. -Element - A singular Metadata element, such as "Author", or "Creator". Elements may have additional metadata, in the form of the associated definition and even a corresponding Dublin Core attribute. See CAS-Metadata for more information on Metadata Elements. -Versioner - A URI generation scheme for Product Types that defines the location within the archive (built by the File Manager) where a file belonging to a Product (that belongs to the associated Product Type) should be placed. -Each Product contains 1 or more References, and one Metadata object. Each Product is a member of a single Product Type. The Metadata collected for each Product is defined by a mapping of Product Type->1...* Elements. Each Product Type has an associated Versioner. These relationships are shown in the below figure. - -File Manager Object Model -Key Capabilities -The File manager has been designed with a new of key capabilities in mind. These capabilities include: - -Easy management of different types of Products. The Repository Manager extension point is responsible for managing Product Types, and their associated information. Management of Product Types includes adding new types, deleting and updating existing types, and retrieving Product Type Objects, by their ID or by their name. - -Support for different kinds of back end catalogs. The Catalog extension point allows Product instance metadata and file location information to be stored in different types of back end data stores quite easily. Existing implementations of the Catalog interface include a JDBC based back end database, along with a flat-file index powered by Lucene. - -Management of Product instance information. Management includes adding, deleting and updating product instance information, including file locations (References), along with Product Metadata. It also includes retrieving Metadata and References associated with existing Products as well as obtaining the Products themselves. - -Element management for Metadata. The File Manager's Validation Layer extension point allows for the management of Element policy information in different types of back end stores. For instance, Element policy could be stored in XML files, a Database, or a Metadata Registry. - -Data transfer mechanism interface. By having an extension point for Data Transfer, the File Manager can support different Data Transfer protocols, both local and remote. - -Advanced support for File Repository layouts. The Versioner extension point allows for different File Repository layouts based on Product Types. - -Support for multiple Product structures. The File Manager Client allows for Products to be Flat, or Hierarchical-based. Flat products are collections of singular files that are aggregated together to make a Product. Hierarchical Products are Products that contain collections of directories, and sub-directories, and files. - -Design for scalability. The File Manager uses the popular client-server paradigm, allowing new File Manager servers to be instantiated, as needed, without affecting the File Manager clients, and vice-versa. - -Standard communication protocols. The File Manager uses XML-RPC as its main external interface between the File Manager client and server. XML-RPC, the little brother of SOAP, is fast, extensible, and uses the underlying HTTP protocol for data transfer. - -RSS-based Product syndication. The File Manager web interface allows for the RSS-based syndication of Product feeds based on Product Type. - -Data transfer status tracking. The File Manager tracks all current Product and File transfers and even publishes an RSS-feed of existing transfers. - -This capability set is not exhaustive, and is meant to give the user a feel for what general features are provided by the File Manager. Most likely the user will find that the File Manager provides many other capabilities besides those described here. - -Extension Points -We have constructed the File Manager making use of the factory method pattern to provide multiple extension points for the File Manager. An extension point is an interface within the File Manager that can have many implementations. This is particularly useful when it comes to software component configuration because it allows different implementations of an existing interface to be selected at deployment time. - -The factory method pattern is a creational pattern common to object oriented design. Each File Manager extension point involves the implementation of two interfaces: an extension factory and an extension implementation. At run-time, the File Manager loads a properties file specifies a factory class to use during extension point instantiation. For example, the File Manager may communicate with a database-based Catalog and an XML-based Element Store (called a Validation Layer), or it may use a Lucene-based Catalog and a database-based Validation Layer. -Using extension points, it is fairly simple to support many different types of what are typically referred to as "plug-in architectures." Each of the core extension points for the File Manager is described below: - -Catalog The Catalog extension point is responsible for storing all the instance data for Products, Metadata, and for file References. Additionally, the Catalog provides a query capability for Products. -Data Transfer The Data Transfer extension point allows for the movement of a Product to and from the archive managed by the File Manager component. Different protocols for Data Transfer may include local (disk-based) copy, or remote XML-RPC based transfer across networked machines. -Repository Manager The Repository Manager extension point provides a means for managing all of the policy information (i.e., the Product Types and their associated information) for Products managed by the File Manager. -Validation Layer The Validation Layer extension point allows for the querying of element definitions associated with a particular Product Type. The extension point also maps Product Type to Elements. -Versioning The Versioning extension point allows for the definition of different URI generation schemes that define the final resting location of files for a particular Product. -System The extension point that provides the external interface to the File Manager services. This includes the File Manager server interface, as well as the associated File Manager client interface, that communicates with the server. -Current Extension Point Implementations -There are at least two implementations of all of the aforementioned extension points for the File Manager. Each extension point implementation is detailed in this section. - -Catalog -Data Source based Catalog. An implementation of the Catalog extension point interface that uses a JDBC accessible database backend. -Lucene based Catalog. An implementation of the Catalog extension point interface that uses the Lucene free text index system to store Product instance information. -Data Transfer -Local Data Transfer. An implementation of the Data Transfer interface that uses Apache's commons-io to perform local, disk based filesystem data transfer. This implementation also supports locally accessible Network File System (NFS) disks. -Remote Data Transfer. An implementation of the Data Transfer interface that uses the XML-RPC File Manager client to transfer files to a remote XML-RPC File Manager server. -InPlace Data Transfer. An implementation of the Data Transfer interface that avoids transfering any products -- this can be used in the situation where metadata about a particular product should be recorded, but no physical transfer needs to occur. -Repository Manager -Data Source based Repository Manager. An implementation of the Repository Manager extension point that stores Product Type policy information in a JDBC accessible database. -XML based Repository Manager. An implementation of the Repository Manager extension point that stores Product Type policy information in an XML file called product-types.xml -Validation Layer -Data Source based Validation Layer. An implementation of the Validation Layer extension point that stores Element policy information in a JDBC accessible database. -XML based Validation Layer. An implementation of the Validation Layer extension point that stores Element policy information in 2 XML files called elements.xml and product-type-element-map.xml -System (File Manager client and File Manager server) -XML-RPC based File Manager server. An implementation of the external server interface for the File Manager that uses XML-RPC as the transportation medium. -XML-RPC based File Manager client. An implementation of the client interface for the XML-RPC File Manager server that uses XML-RPC as the transportation medium. -Use Cases -The File Manager was built to support several of the above capabilities outlined in Section 3. In particular there were several use cases that we wanted to support, some of which are described below. - -File Manager Ingest Use Case -The red numbers in the above Figure correspond to a sequence of steps that occurs and a series of interactions between the different File Manager extension points in order to perform the file ingestion activity. In Step 1, a File Manager client is invoked for the ingest operation, which sends Metadata and References for a particular Product to ingest to the File Manager server’s System Interface extension point. The System Interface uses the information about Product Type policy made available by the Repository Manager in order to understand whether or not the product should be transferred, where it’s root repository path should be, and so on. The System Interface then catalogs the file References and Metadata using the Catalog extension point. During this catalog process, the Catalog extension point uses the Validation Layer to determine which Elements should be extracted for the particular Product, based upon its Product Type. After that, Data Transfer is initiated either at the client or server end, and the first step to Data Transfer is using the Product’s associated Versioner to generate final file References. After final file References have been determined, the file data is transferred by the server or by the client, using the Data Transfer extension point. - -Conclusion -The aim of this document is to provide information relevant to developers about the CAS File Manager. Specifically, this document has described the File Manager's architecture, including its constituent components, object model and key capabilities. Additionally, the this document provides an overview of the current implementations of the File Manager's extension points. \ No newline at end of file diff --git "a/src/main/resources/sdtocode/doc/Apache OODT File Manager/cas-filemgr \342\200\223 CAS File Manager Developer Guide.txt.xml.xls" "b/src/main/resources/sdtocode/doc/Apache OODT File Manager/cas-filemgr \342\200\223 CAS File Manager Developer Guide.txt.xml.xls" deleted file mode 100644 index 47ee0175b2b0f2f838986e32b86fae0973f3d173..0000000000000000000000000000000000000000 Binary files "a/src/main/resources/sdtocode/doc/Apache OODT File Manager/cas-filemgr \342\200\223 CAS File Manager Developer Guide.txt.xml.xls" and /dev/null differ diff --git a/src/main/resources/sdtocode/sd/Fig1SeqDiagram.jfif b/src/main/resources/sdtocode/sd/Fig1SeqDiagram.jfif deleted file mode 100644 index a234ace3796dcfeb71e5a3418ad55c6c55ce82d5..0000000000000000000000000000000000000000 Binary files a/src/main/resources/sdtocode/sd/Fig1SeqDiagram.jfif and /dev/null differ diff --git a/src/main/resources/sdtocode/sd/Fig2SeqDiagram.jfif b/src/main/resources/sdtocode/sd/Fig2SeqDiagram.jfif deleted file mode 100644 index 262de8840f084b7f323787dd5012db5dc2fb55b0..0000000000000000000000000000000000000000 Binary files a/src/main/resources/sdtocode/sd/Fig2SeqDiagram.jfif and /dev/null differ diff --git a/src/main/resources/sdtocode/sd/sd-Hadoop HDFS1.png b/src/main/resources/sdtocode/sd/sd-Hadoop HDFS1.png new file mode 100644 index 0000000000000000000000000000000000000000..eb03bf11d1fc61f274658e4d2ab6f2b4e020571e Binary files /dev/null and b/src/main/resources/sdtocode/sd/sd-Hadoop HDFS1.png differ diff --git a/src/main/resources/sdtocode/sd/sd-Hadoop HDFS1.txt b/src/main/resources/sdtocode/sd/sd-Hadoop HDFS1.txt new file mode 100644 index 0000000000000000000000000000000000000000..729651169b560505a20ca0d6efe74e82a3c172f9 --- /dev/null +++ b/src/main/resources/sdtocode/sd/sd-Hadoop HDFS1.txt @@ -0,0 +1,18 @@ +DFSOutputStream +@DFSOutputStream +%NameNode +%addBlock() +￥DFSOutputStream +%DataNode +%opWriteBlock()￥ +@#NameNode +@NameNode +%DFSOutputStream +%return nodes +￥@#DataNode +@DataNode +%NameNode +%blockReceived +￥DataNode +%DFSOutputStream +%￥@# \ No newline at end of file diff --git a/src/main/resources/sdtocode/sd/sd-Hadoop HDFS2.png b/src/main/resources/sdtocode/sd/sd-Hadoop HDFS2.png new file mode 100644 index 0000000000000000000000000000000000000000..5d30e2d09a390ab2d54f629c38b25f26af142919 Binary files /dev/null and b/src/main/resources/sdtocode/sd/sd-Hadoop HDFS2.png differ diff --git a/src/main/resources/sdtocode/sd/sd-Hadoop HDFS2.txt b/src/main/resources/sdtocode/sd/sd-Hadoop HDFS2.txt new file mode 100644 index 0000000000000000000000000000000000000000..c187e10d618e79b14b9b283370a7abba83039b52 --- /dev/null +++ b/src/main/resources/sdtocode/sd/sd-Hadoop HDFS2.txt @@ -0,0 +1,38 @@ +DFSOutputStream +@DFSOutputStream +%DataStreamer +%run() +￥@#DataStreamer +@DataStreamer +%DataNodeInfo +%nextBlockOutputStream() +￥DataStreamer +%DataNode +%DataTransferProtocol.Sender.opWriteBlock() +￥@#DataNodeInfo +@DataNodeInfo +%LocatedBlock +%locateFollowingBlock() +￥DataNodeInfo +%DataStreamer +%￥@#LocatedBlock +@LocatedBlock +%NameNode +%dfsClient.namenode.addBlock() +￥LocatedBlock +%DataNodeInfo +%return LocatedBlock +￥@#NameNode +@NameNode +%LocatedBlock +%return LocatedBlock +￥@#DataNode +@DataNode +%DataTransferProtocol.Sender +%write +￥@#DataTransferProtocol.Sender +@DataTransferProtocol.Sender +%DataStreamer +%￥DataTransferProtocol.Sender +%NameNode +%blockReceived￥@# \ No newline at end of file diff --git a/src/main/resources/sdtocode/sd/sd-Hadoop MapReduce.jpg b/src/main/resources/sdtocode/sd/sd-Hadoop MapReduce.jpg new file mode 100644 index 0000000000000000000000000000000000000000..39c4bd1b52d1ea42b6a9041dd1dae8413302e5a5 Binary files /dev/null and b/src/main/resources/sdtocode/sd/sd-Hadoop MapReduce.jpg differ diff --git a/src/main/resources/sdtocode/sd/sd-Hadoop MapReduce.txt b/src/main/resources/sdtocode/sd/sd-Hadoop MapReduce.txt new file mode 100644 index 0000000000000000000000000000000000000000..3735143fa8d19bf79a1d8924b63bed455c0a139c --- /dev/null +++ b/src/main/resources/sdtocode/sd/sd-Hadoop MapReduce.txt @@ -0,0 +1,25 @@ +JobClient +@JobClient +%JobTracker +%getNewJobId() +￥JobClient +%HDFS +%mkdirs() +￥JobClient +%HDFS +%copyRemoteFiles() +￥JobClient +%HDFS +%writeSplits() +￥JobClient +%HDFS +%writeXml() +￥JobClient +%HDFS +%submitJob(job) +￥@#JobTracker +@JobTracker +%JobClient +%jobId +￥@#HDFS +@@# \ No newline at end of file diff --git a/src/test/java/com/hy/java/uct/cdtocode/CodeRelationMapperTest.java b/src/test/java/com/hy/java/uct/cdtocode/CodeRelationMapperTest.java index e6786eb6fea7514a3410374fc395fc3baf4f281a..8937aaa0560742c9876165c15b6f016f01398a70 100644 --- a/src/test/java/com/hy/java/uct/cdtocode/CodeRelationMapperTest.java +++ b/src/test/java/com/hy/java/uct/cdtocode/CodeRelationMapperTest.java @@ -26,8 +26,8 @@ import com.github.javaparser.ast.type.ClassOrInterfaceType; import com.hy.java.uct.cdtocode.reader.CDReader; import com.hy.java.uct.cdtocode.reader.CodeReader; import com.hy.java.uct.cdtocode.util.MappedFile; -import com.hy.java.uct.util.ImgRelation; -import com.hy.java.uct.util.UMLClass; +import com.hy.java.uct.util.cd.ImgRelation; +import com.hy.java.uct.util.cd.UMLClass; import com.hy.java.utility.common.FileEditor; import com.hy.java.utility.common.Pair; diff --git a/src/test/java/com/hy/java/uct/cdtocode/DocAnalyzerTest.java b/src/test/java/com/hy/java/uct/cdtocode/DocAnalyzerTest.java index 24a52225ddd16081f762f7a110d90de1a5519018..c3cb946f3203f3137e4ed10c5b7b6c11d055d066 100644 --- a/src/test/java/com/hy/java/uct/cdtocode/DocAnalyzerTest.java +++ b/src/test/java/com/hy/java/uct/cdtocode/DocAnalyzerTest.java @@ -14,7 +14,7 @@ import org.junit.jupiter.api.Test; import com.hy.java.uct.cdtocode.reader.CDReader; import com.hy.java.uct.cdtocode.reader.DocReader; -import com.hy.java.uct.util.UMLClass; +import com.hy.java.uct.util.cd.UMLClass; import edu.stanford.nlp.pipeline.CoreDocument; import edu.stanford.nlp.pipeline.CoreSentence;