Added extra functionality (generation of tool_conf.xml, support of several inputs, etc.)
Luis de la Garza
9 years ago
| 7 | 7 | import sys |
| 8 | 8 | import os |
| 9 | 9 | import traceback |
| 10 | import ntpath | |
| 10 | 11 | |
| 11 | 12 | from argparse import ArgumentParser |
| 12 | 13 | from argparse import RawDescriptionHelpFormatter |
| 33 | 34 | def __unicode__(self): |
| 34 | 35 | return self.msg |
| 35 | 36 | |
| 37 | class ApplicationException(Exception): | |
| 38 | def __init__(self, msg): | |
| 39 | super(ApplicationException).__init__(type(self)) | |
| 40 | self.msg = msg | |
| 41 | def __str__(self): | |
| 42 | return self.msg | |
| 43 | def __unicode__(self): | |
| 44 | return self.msg | |
| 45 | ||
| 36 | 46 | class ExitCode: |
| 37 | 47 | def __init__(self, code_range="", level="", description=""): |
| 38 | 48 | self.range = code_range |
| 84 | 94 | try: |
| 85 | 95 | # Setup argument parser |
| 86 | 96 | parser = ArgumentParser(prog="GalaxyConfigGenerator", description=program_license, formatter_class=RawDescriptionHelpFormatter, add_help=True) |
| 87 | parser.add_argument("-i", "--input-file", dest="input_file", help="provide a single input CTD file to convert", required=True) | |
| 88 | parser.add_argument("-o", "--output-file", dest="output_file", help="provide a single output Galaxy wrapper file", required=True) | |
| 97 | parser.add_argument("-i", "--input", dest="input_files", required=True, nargs="+", action="append", | |
| 98 | help="list of CTD files to convert.") | |
| 99 | parser.add_argument("-o", "--output-destination", dest="output_dest", required=True, | |
| 100 | help="if multiple input files are given, then a folder in which all generated XMLs will generated is expected;"\ | |
| 101 | "if a single input file is given, then a destination file is expected.") | |
| 89 | 102 | parser.add_argument("-a", "--add-to-command-line", dest="add_to_command_line", help="adds content to the command line", default="", required=False) |
| 90 | 103 | parser.add_argument("-w", "--whitespace-validation", dest="whitespace_validation", action="store_true", default=False, |
| 91 | 104 | help="if true, each parameter in the generated command line will be "+ |
| 99 | 112 | parser.add_argument("-x", "--exit-code", dest="exit_codes", default=[], nargs="+", action="append", |
| 100 | 113 | help="list of <stdio> galaxy exit codes, in the following format: range=<range>,level=<level>,description=<description>,\n" + |
| 101 | 114 | "example: --exit-codes \"range=3:4,level=fatal,description=Out of memory\"") |
| 115 | parser.add_argument("-t", "--tool-conf-destination", dest="tool_conf_dest", default=None, required=False, | |
| 116 | help="specify the destination file of a generated tool_conf.xml for all given input files; each category will be written in its own section.") | |
| 117 | parser.add_argument("-g", "--galaxy-tool-path", dest="galaxy_tool_path", default=None, required=False, | |
| 118 | help="the path that will be prepended to the file names when generating tool_conf.xml") | |
| 102 | 119 | # verbosity will be added later on, will not waste time on this now |
| 103 | 120 | # parser.add_argument("-v", "--verbose", dest="verbose", action="count", help="set verbosity level [default: %(default)s]") |
| 104 | 121 | parser.add_argument("-V", "--version", action='version', version=program_version_message) |
| 106 | 123 | # Process arguments |
| 107 | 124 | args = parser.parse_args() |
| 108 | 125 | |
| 109 | # collect arguments | |
| 110 | input_file = args.input_file | |
| 111 | output_file = args.output_file | |
| 126 | # validate and prepare the passed arguments | |
| 127 | validate_and_prepare_args(args) | |
| 112 | 128 | |
| 113 | 129 | #if verbose > 0: |
| 114 | 130 | # print("Verbose mode on") |
| 115 | convert(input_file, | |
| 116 | output_file, | |
| 131 | convert(args.input_files, | |
| 132 | args.output_dest, | |
| 117 | 133 | add_to_command_line=args.add_to_command_line, |
| 118 | 134 | whitespace_validation=args.whitespace_validation, |
| 119 | 135 | quote_parameters=args.quote_parameters, |
| 120 | 136 | # remember that blacklisted_parameters, package_requirements and exit_codes are lists of lists of strings |
| 121 | blacklisted_parameters=[item for sublist in args.blacklisted_parameters for item in sublist], | |
| 122 | package_requirements=[item for sublist in args.package_requirements for item in sublist], | |
| 123 | exit_codes=convert_exit_codes([item for sublist in args.exit_codes for item in sublist])) | |
| 137 | blacklisted_parameters=args.blacklisted_parameters, | |
| 138 | package_requirements=args.package_requirements, | |
| 139 | exit_codes=args.exit_codes, | |
| 140 | galaxy_tool_path=args.galaxy_tool_path, | |
| 141 | tool_conf_dest=args.tool_conf_dest) | |
| 124 | 142 | return 0 |
| 125 | 143 | |
| 126 | 144 | except KeyboardInterrupt: |
| 127 | 145 | ### handle keyboard interrupt ### |
| 128 | 146 | return 0 |
| 129 | except IOError, e: | |
| 130 | indent = len(program_name) * " " | |
| 131 | sys.stderr.write(program_name + ": " + repr(e) + "\n") | |
| 132 | sys.stderr.write(indent + "Could not access input file [%(input)s] or output file [%(output)s]\n" % {"input":input_file, "output":output_file}) | |
| 133 | sys.stderr.write(indent + "For help use --help\n") | |
| 134 | # #define EX_NOINPUT 66 /* cannot open input */ | |
| 135 | return 66 | |
| 147 | except ApplicationException, e: | |
| 148 | sys.stderr.write("GalaxyConfigGenerator could not complete the requested operation.\n") | |
| 149 | sys.stderr.write("Reason: " + e.msg) | |
| 150 | return 1 | |
| 136 | 151 | except ModelError, e: |
| 137 | 152 | indent = len(program_name) * " " |
| 138 | 153 | sys.stderr.write(program_name + ": " + repr(e) + "\n") |
| 139 | sys.stderr.write(indent + "There seems to be a problem with your input CTD [%s], please make sure that it is a valid CTD.\n" % input_file) | |
| 154 | sys.stderr.write(indent + "There seems to be a problem with one your input CTD.\n") | |
| 140 | 155 | sys.stderr.write(indent + "For help use --help\n") |
| 141 | 156 | return 1 |
| 142 | 157 | except Exception, e: |
| 143 | 158 | traceback.print_exc() |
| 144 | 159 | return 2 |
| 160 | ||
| 161 | def validate_and_prepare_args(args): | |
| 162 | # first, we convert all list of lists to flat lists | |
| 163 | args.input_files = [item for sublist in args.input_files for item in sublist] | |
| 164 | args.blacklisted_parameters=[item for sublist in args.blacklisted_parameters for item in sublist] | |
| 165 | args.package_requirements=[item for sublist in args.package_requirements for item in sublist] | |
| 166 | args.exit_codes=convert_exit_codes([item for sublist in args.exit_codes for item in sublist]) | |
| 167 | ||
| 168 | # if input is a single file, we expect output to be a file (and not a dir that already exists) | |
| 169 | if len(args.input_files) == 1: | |
| 170 | if os.path.isdir(args.output_dest): | |
| 171 | raise ApplicationException("If a single input file is provided, output (%s) is expected to be a file and not a folder." % args.output_dest) | |
| 172 | ||
| 173 | # if input is a list of files, we expect output to be a folder | |
| 174 | if len(args.input_files) > 1: | |
| 175 | if not os.path.isdir(args.output_dest): | |
| 176 | raise ApplicationException("If several input files are provided, output (%s) is expected to be an existing directory." % args.output_dest) | |
| 145 | 177 | |
| 146 | 178 | def convert_exit_codes(exit_codes_raw): |
| 147 | 179 | # input is in the format: |
| 157 | 189 | exit_codes.append(exit_code) |
| 158 | 190 | return exit_codes |
| 159 | 191 | |
| 160 | def convert(input_file, output_file, **kwargs): | |
| 192 | def convert(input_files, output_dest, **kwargs): | |
| 161 | 193 | # first, generate a model |
| 162 | print("Parsing CTD from [%s]" % input_file) | |
| 163 | model = CTDModel(from_file=input_file) | |
| 164 | ||
| 194 | is_converting_multiple_ctds = len(input_files) > 1 | |
| 195 | parsed_models = [] | |
| 196 | try: | |
| 197 | for input_file in input_files: | |
| 198 | print("Parsing CTD from [%s]" % input_file) | |
| 199 | model = CTDModel(from_file=input_file) | |
| 200 | ||
| 201 | doc = Document() | |
| 202 | tool = create_tool(doc, model) | |
| 203 | doc.appendChild(tool) | |
| 204 | create_description(doc, tool, model) | |
| 205 | create_requirements(doc, tool, model, kwargs["package_requirements"]) | |
| 206 | create_command(doc, tool, model, **kwargs) | |
| 207 | create_inputs(doc, tool, model, kwargs["blacklisted_parameters"]) | |
| 208 | create_outputs(doc, tool, model, kwargs["blacklisted_parameters"]) | |
| 209 | create_exit_codes(doc, tool, model, kwargs["exit_codes"]) | |
| 210 | create_help(doc, tool, model) | |
| 211 | ||
| 212 | # finally, serialize the tool | |
| 213 | output_file = output_dest | |
| 214 | # if multiple inputs are being converted, then we need to generate a different output_file for each input | |
| 215 | if is_converting_multiple_ctds: | |
| 216 | if not output_file.endswith('/'): | |
| 217 | output_file += "/" | |
| 218 | output_file += get_filename(input_file) + ".xml" | |
| 219 | doc.writexml(open(output_file, 'w'), indent=" ", addindent=" ", newl='\n') | |
| 220 | # let's use model to hold the name of the outputfile | |
| 221 | parsed_models.append([model, get_filename(output_file)]) | |
| 222 | print("Generated Galaxy wrapper in [%s]\n" % output_file) | |
| 223 | # generation of galaxy stubs is ready... now, let's see if we need to generate a tool_conf.xml | |
| 224 | if kwargs["tool_conf_dest"] is not None: | |
| 225 | generate_tool_conf(parsed_models, kwargs["tool_conf_dest"], kwargs["galaxy_tool_path"]) | |
| 226 | ||
| 227 | except IOError, e: | |
| 228 | raise ApplicationException("One of the provided input files or the destination file could not be accessed. Detailed information: " + str(e) + "\n") | |
| 229 | ||
| 230 | def generate_tool_conf(parsed_models, tool_conf_dest, galaxy_tool_path): | |
| 231 | # for each category, we keep a list of models corresponding to it | |
| 232 | categories_to_tools = dict() | |
| 233 | for model in parsed_models: | |
| 234 | if "category" in model[0].opt_attribs: | |
| 235 | category = model[0].opt_attribs["category"] | |
| 236 | if category is not None and len(strip(category)) > 0: | |
| 237 | category = strip(category) | |
| 238 | if category not in categories_to_tools: | |
| 239 | categories_to_tools[category] = [] | |
| 240 | categories_to_tools[category].append(model[1]) | |
| 241 | ||
| 242 | # at this point, we should have a map for all categories->tools | |
| 165 | 243 | doc = Document() |
| 166 | tool = create_tool(doc, model) | |
| 167 | doc.appendChild(tool) | |
| 168 | create_description(doc, tool, model) | |
| 169 | create_requirements(doc, tool, model, kwargs["package_requirements"]) | |
| 170 | create_command(doc, tool, model, **kwargs) | |
| 171 | create_inputs(doc, tool, model, kwargs["blacklisted_parameters"]) | |
| 172 | create_outputs(doc, tool, model, kwargs["blacklisted_parameters"]) | |
| 173 | create_exit_codes(doc, tool, model, kwargs["exit_codes"]) | |
| 174 | create_help(doc, tool, model) | |
| 175 | ||
| 176 | # finally, serialize the tool | |
| 177 | doc.writexml(open(output_file, 'w'), indent=" ", addindent=" ", newl='\n') | |
| 178 | print("Generated Galaxy wrapper in [%s]\n" % output_file) | |
| 244 | toolbox_node = doc.createElement("toolbox") | |
| 245 | ||
| 246 | if galaxy_tool_path is not None and not galaxy_tool_path.strip().endswith("/"): | |
| 247 | galaxy_tool_path = galaxy_tool_path.strip() + "/" | |
| 248 | if galaxy_tool_path is None: | |
| 249 | galaxy_tool_path = "" | |
| 250 | ||
| 251 | for category, filenames in categories_to_tools.iteritems(): | |
| 252 | section_node = doc.createElement("section") | |
| 253 | section_node.setAttribute("id", "section-id-" + "".join(category.split())) | |
| 254 | section_node.setAttribute("name", category) | |
| 255 | ||
| 256 | for filename in filenames: | |
| 257 | tool_node = doc.createElement("tool") | |
| 258 | tool_node.setAttribute("file", galaxy_tool_path + filename) | |
| 259 | toolbox_node.appendChild(section_node) | |
| 260 | section_node.appendChild(tool_node) | |
| 261 | toolbox_node.appendChild(section_node) | |
| 262 | ||
| 263 | doc.appendChild(toolbox_node) | |
| 264 | doc.writexml(open(tool_conf_dest, 'w'), indent=" ", addindent=" ", newl='\n') | |
| 265 | print("Generated Galaxy tool_conf.xml in [%s]\n" % tool_conf_dest) | |
| 266 | ||
| 267 | # taken from | |
| 268 | # http://stackoverflow.com/questions/8384737/python-extract-file-name-from-path-no-matter-what-the-os-path-format | |
| 269 | def get_filename(path): | |
| 270 | head, tail = ntpath.split(path) | |
| 271 | return tail or ntpath.basename(head) | |
| 179 | 272 | |
| 180 | 273 | def create_tool(doc, model): |
| 181 | 274 | tool = doc.createElement("tool") |
| 352 | 445 | # we ASSUME that a list of parameters looks like: |
| 353 | 446 | # $ tool -ignore He Ar Xe |
| 354 | 447 | # meaning, that, for example, Helium, Argon and Xenon will be ignored |
| 355 | param_node.setAttribute("value", ' '.join(param.default)) | |
| 448 | param_node.setAttribute("value", ' '.join(map(str, param.default))) | |
| 356 | 449 | elif param_type != "boolean": |
| 357 | 450 | # boolean parameters handle default values by using the "checked" attribute |
| 358 | 451 | # there isn't much we can do... just stringify the value |
| 362 | 455 | # galaxy requires "value" to be included for int/float |
| 363 | 456 | # since no default was included, we need to figure out one in a clever way... but let the user know |
| 364 | 457 | # that we are "thinking" for him/her |
| 365 | warning("Generating default value for parameter [%s]. Galaxy requires the attribute 'value' to be set for integer/floats."\ | |
| 366 | "You might want to edit the CTD file and provide a suitable default value." % param.name) | |
| 458 | warning("Generating default value for parameter [%s]. Galaxy requires the attribute 'value' to be set for integer/floats. "\ | |
| 459 | "Edit the CTD file and provide a suitable default value." % param.name) | |
| 367 | 460 | # check if there's a min/max and try to use them |
| 368 | 461 | default_value = None |
| 369 | if type(param.restrictions) is _NumericRange: | |
| 370 | default_value = param.restrictions.n_min | |
| 371 | if default_value is None: | |
| 372 | default_value = param.restrictions.n_max | |
| 373 | if default_value is None: | |
| 374 | # no min/max provided... just use 0 and see what happens | |
| 375 | default_value = 0 | |
| 462 | if param.restrictions is not None: | |
| 463 | if type(param.restrictions) is _NumericRange: | |
| 464 | default_value = param.restrictions.n_min | |
| 465 | if default_value is None: | |
| 466 | default_value = param.restrictions.n_max | |
| 467 | if default_value is None: | |
| 468 | # no min/max provided... just use 0 and see what happens | |
| 469 | default_value = 0 | |
| 470 | else: | |
| 471 | # should never be here, since we have validated this anyway... this code is here just for documentation purposes | |
| 472 | # however, better safe than sorry! (it could be that the code changes and then we have an ugly scenario) | |
| 473 | raise InvalidModelException("Expected either a numeric range for parameter [%(name)s], but instead got [%(type)s]" % {"name":param.name, "type":type(param.restrictions)}) | |
| 376 | 474 | else: |
| 377 | # should never be here, since we have validated this anyway... this code is here just for documentation purposes | |
| 378 | # however, better safe than sorry! (it could be that the code changes and then we have an ugly scenario) | |
| 379 | raise InvalidModelException("Expected either a numeric range for parameter [%(name)s], but instead got [%(type)s]" % {"name":param.name, "type":type(param.restrictions)}) | |
| 475 | # no restrictions and no default value provided... | |
| 476 | # make up something | |
| 477 | default_value = 0 | |
| 380 | 478 | param_node.setAttribute("value", str(default_value)) |
| 381 | 479 | |
| 382 | 480 | return param_node |