Package list ctdconverter / 7d1d48c galaxy / generator.py
7d1d48c

Tree @7d1d48c (Download .tar.gz)

generator.py @7d1d48craw · history · blame

   1
   2
   3
   4
   5
   6
   7
   8
   9
  10
  11
  12
  13
  14
  15
  16
  17
  18
  19
  20
  21
  22
  23
  24
  25
  26
  27
  28
  29
  30
  31
  32
  33
  34
  35
  36
  37
  38
  39
  40
  41
  42
  43
  44
  45
  46
  47
  48
  49
  50
  51
  52
  53
  54
  55
  56
  57
  58
  59
  60
  61
  62
  63
  64
  65
  66
  67
  68
  69
  70
  71
  72
  73
  74
  75
  76
  77
  78
  79
  80
  81
  82
  83
  84
  85
  86
  87
  88
  89
  90
  91
  92
  93
  94
  95
  96
  97
  98
  99
 100
 101
 102
 103
 104
 105
 106
 107
 108
 109
 110
 111
 112
 113
 114
 115
 116
 117
 118
 119
 120
 121
 122
 123
 124
 125
 126
 127
 128
 129
 130
 131
 132
 133
 134
 135
 136
 137
 138
 139
 140
 141
 142
 143
 144
 145
 146
 147
 148
 149
 150
 151
 152
 153
 154
 155
 156
 157
 158
 159
 160
 161
 162
 163
 164
 165
 166
 167
 168
 169
 170
 171
 172
 173
 174
 175
 176
 177
 178
 179
 180
 181
 182
 183
 184
 185
 186
 187
 188
 189
 190
 191
 192
 193
 194
 195
 196
 197
 198
 199
 200
 201
 202
 203
 204
 205
 206
 207
 208
 209
 210
 211
 212
 213
 214
 215
 216
 217
 218
 219
 220
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 351
 352
 353
 354
 355
 356
 357
 358
 359
 360
 361
 362
 363
 364
 365
 366
 367
 368
 369
 370
 371
 372
 373
 374
 375
 376
 377
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394
 395
 396
 397
 398
 399
 400
 401
 402
 403
 404
 405
 406
 407
 408
 409
 410
 411
 412
 413
 414
 415
 416
 417
 418
 419
 420
 421
 422
 423
 424
 425
 426
 427
 428
 429
 430
 431
 432
 433
 434
 435
 436
 437
 438
 439
 440
 441
 442
 443
 444
 445
 446
 447
 448
 449
 450
 451
 452
 453
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469
 470
 471
 472
 473
 474
 475
 476
 477
 478
 479
 480
 481
 482
 483
 484
 485
 486
 487
 488
 489
 490
 491
 492
 493
 494
 495
 496
 497
 498
 499
 500
 501
 502
 503
 504
 505
 506
 507
 508
 509
 510
 511
 512
 513
 514
 515
 516
 517
 518
 519
 520
 521
 522
 523
 524
 525
 526
 527
 528
 529
 530
 531
 532
 533
 534
 535
 536
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557
 558
 559
 560
 561
 562
 563
 564
 565
 566
 567
 568
 569
 570
 571
 572
 573
 574
 575
 576
 577
 578
 579
 580
 581
 582
 583
 584
 585
 586
 587
 588
 589
 590
 591
 592
 593
 594
 595
 596
 597
 598
 599
 600
 601
 602
 603
 604
 605
 606
 607
 608
 609
 610
 611
 612
 613
 614
 615
 616
 617
 618
 619
 620
 621
 622
 623
 624
 625
 626
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
#!/usr/bin/env python
# encoding: utf-8

"""
@author:     delagarza
"""


import sys
import os
import traceback
import ntpath
import string

from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter
from collections import OrderedDict
from string import strip
from lxml import etree
from lxml.etree import SubElement, Element, ElementTree, ParseError, parse

from CTDopts.CTDopts import CTDModel, _InFile, _OutFile, ParameterGroup, _Choices, _NumericRange, _FileFormat, \
    ModelError, _Null

__all__ = []
__version__ = 1.0
__date__ = '2014-09-17'
__updated__ = '2016-05-09'

MESSAGE_INDENTATION_INCREMENT = 2

TYPE_TO_GALAXY_TYPE = {int: 'integer', float: 'float', str: 'text', bool: 'boolean', _InFile: 'data', 
                       _OutFile: 'data', _Choices: 'select'}

STDIO_MACRO_NAME = "stdio"
REQUIREMENTS_MACRO_NAME = "requirements"
ADVANCED_OPTIONS_MACRO_NAME = "advanced_options"

REQUIRED_MACROS = [STDIO_MACRO_NAME, REQUIREMENTS_MACRO_NAME, ADVANCED_OPTIONS_MACRO_NAME]


class CLIError(Exception):
    # Generic exception to raise and log different fatal errors.
    def __init__(self, msg):
        super(CLIError).__init__(type(self))
        self.msg = "E: %s" % msg

    def __str__(self):
        return self.msg

    def __unicode__(self):
        return self.msg


class InvalidModelException(ModelError):
    def __init__(self, message):
        super(InvalidModelException, self).__init__()
        self.message = message

    def __str__(self):
        return self.message

    def __repr__(self):
        return self.message


class ApplicationException(Exception):
    def __init__(self, msg):
        super(ApplicationException).__init__(type(self))
        self.msg = msg

    def __str__(self):
        return self.msg

    def __unicode__(self):
        return self.msg


class ExitCode:
    def __init__(self, code_range="", level="", description=None):
        self.range = code_range
        self.level = level
        self.description = description    


class DataType:
    def __init__(self, extension, galaxy_extension=None, galaxy_type=None, mimetype=None):
        self.extension = extension
        self.galaxy_extension = galaxy_extension
        self.galaxy_type = galaxy_type
        self.mimetype = mimetype


class ParameterHardcoder:
    def __init__(self):
        # map whose keys are the composite names of tools and parameters in the following pattern:
        # [ToolName][separator][ParameterName] -> HardcodedValue
        # if the parameter applies to all tools, then the following pattern is used:
        # [ParameterName] -> HardcodedValue

        # examples (assuming separator is '#'):
        # threads -> 24
        # XtandemAdapter#adapter -> xtandem.exe
        # adapter -> adapter.exe
        self.separator = "!"
        self.parameter_map = {}

    # the most specific value will be returned in case of overlap
    def get_hardcoded_value(self, parameter_name, tool_name):
        # look for the value that would apply for all tools
        generic_value = self.parameter_map.get(parameter_name, None)
        specific_value = self.parameter_map.get(self.build_key(parameter_name, tool_name), None)
        if specific_value is not None:
            return specific_value

        return generic_value

    def register_parameter(self, parameter_name, parameter_value, tool_name=None):
        self.parameter_map[self.build_key(parameter_name, tool_name)] = parameter_value

    def build_key(self, parameter_name, tool_name):
        if tool_name is None:
            return parameter_name
        return "%s%s%s" % (parameter_name, self.separator, tool_name)


def main(argv=None):  # IGNORE:C0111
    # Command line options.
    if argv is None:
        argv = sys.argv
    else:
        sys.argv.extend(argv)

    program_version = "v%s" % __version__
    program_build_date = str(__updated__)
    program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date)
    program_short_description = "CTD2Galaxy - A project from the GenericWorkflowNodes family " \
                                "(https://github.com/orgs/genericworkflownodes)"
    program_usage = '''
    USAGE:
    
    I - Parsing a single CTD file and generate a Galaxy wrapper:

    $ python generator.py -i input.ctd -o output.xml
    

    II - Parsing all found CTD files (files with .ctd and .xml extension) in a given folder and
         output converted Galaxy wrappers in a given folder:

    $ python generator.py -i /home/user/*.ctd -o /home/user/galaxywrappers


    III - Providing file formats, mimetypes

    Galaxy supports the concept of file format in order to connect compatible ports, that is, input ports of a certain
    data format will be able to receive data from a port from the same format. This converter allows you to provide
    a personalized file in which you can relate the CTD data formats with supported Galaxy data formats. The layout of
    this file consists of lines, each of either one or four columns separated by any amount of whitespace. The content
    of each column is as follows:

    * 1st column: file extension
    * 2nd column: data type, as listed in Galaxy
    * 3rd column: full-named Galaxy data type, as it will appear on datatypes_conf.xml
    * 4th column: mimetype (optional)

    The following is an example of a valid "file formats" file:

    ########################################## FILE FORMATS example ##########################################
    # Every line starting with a # will be handled as a comment and will not be parsed.
    # The first column is the file format as given in the CTD and second column is the Galaxy data format.
    # The second, third, fourth and fifth column can be left empty if the data type has already been registered
    # in Galaxy, otherwise, all but the mimetype must be provided.

    # CTD type    # Galaxy type      # Long Galaxy data type            # Mimetype
    csv           tabular            galaxy.datatypes.data:Text
    fasta
    ini           txt                galaxy.datatypes.data:Text
    txt
    idxml         txt                galaxy.datatypes.xml:GenericXml    application/xml
    options       txt                galaxy.datatypes.data:Text
    grid          grid               galaxy.datatypes.data:Grid

    ##########################################################################################################

    Note that each line consists precisely of either one, three or four columns. In the case of data types already
    registered in Galaxy (such as fasta and txt in the above example), only the first column is needed. In the case of
    data types that haven't been yet registered in Galaxy, the first three columns are needed (mimetype is optional).

    For information about Galaxy data types and subclasses, see the following page:
    https://wiki.galaxyproject.org/Admin/Datatypes/Adding%20Datatypes


    IV - Hardcoding parameters

    It is possible to hardcode parameters. This makes sense if you want to set a tool in Galaxy in 'quiet' mode or if
    your tools support multi-threading and accept the number of threads via a parameter, without giving the end user the
    chance to change the values for these parameters.

    In order to generate hardcoded parameters, you need to provide a simple file. Each line of this file contains two
    or three columns separated by whitespace. Any line starting with a '#' will be ignored. The first column contains
    the name of the parameter, the second column contains the value that will always be set for this parameter. The
    first two columns are mandatory.

    If the parameter is to be hardcoded only for a set of tools, then a third column can be added. This column includes
    a comma-separated list of tool names for which the parameter will be hardcoded. If a third column is not included,
    then all processed tools containing the given parameter will get a hardcoded value for it.

    The following is an example of a valid file:

    ##################################### HARDCODED PARAMETERS example #####################################
    # Every line starting with a # will be handled as a comment and will not be parsed.
    # The first column is the name of the parameter and the second column is the value that will be used.

    # Parameter name            # Value                     # Tool(s)
    threads                     \${GALAXY_SLOTS:-24}
    mode                        quiet
    xtandem_executable          xtandem                     XTandemAdapter
    verbosity                   high                        Foo, Bar

    #########################################################################################################

    Using the above file will produce a <command> similar to:

    [tool_name] ... -threads \${GALAXY_SLOTS:-24} -mode quiet ...

    For all tools. For XTandemAdapter, the <command> will be similar to:

    XtandemAdapter ... -threads \${GALAXY_SLOTS:-24} -mode quiet -xtandem_executable xtandem ...

    And for tools Foo and Bar, the <command> will be similar to:

    Foo ... ... -threads \${GALAXY_SLOTS:-24} -mode quiet -verbosity high ...


    V - Control which tools will be converted

    Sometimes only a subset of CTDs needs to be converted. It is possible to either explicitly specify which tools will
    be converted or which tools will not be converted.

    The value of the -s/--skip-tools parameter is a file in which each line will be interpreted as the name of a tool
    that will not be converted. Conversely, the value of the -r/--required-tools is a file in which each line will be
    interpreted as a tool that is required. Only one of these parameters can be specified at a given time.

    The format of both files is exactly the same. As stated before, each line will be interpreted as the name of a tool;
    any line starting with a '#' will be ignored.

    '''
    program_license = '''%(short_description)s
    Copyright 2015, Luis de la Garza

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

        http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
    
    %(usage)s
''' % {'short_description': program_short_description, 'usage': program_usage}

    try:
        # Setup argument parser
        parser = ArgumentParser(prog="CTD2Galaxy", description=program_license,
                                formatter_class=RawDescriptionHelpFormatter, add_help=True)
        parser.add_argument("-i", "--input", dest="input_files", default=[], required=True, nargs="+", action="append",
                            help="List of CTD files to convert.")
        parser.add_argument("-o", "--output-destination", dest="output_destination", required=True,
                            help="If multiple input files are given, then a folder in which all generated "
                                 "XMLs will be generated is expected;"
                                 "if a single input file is given, then a destination file is expected.")
        parser.add_argument("-f", "--formats-file", dest="formats_file",
                            help="File containing the supported file formats. Run with '-h' or '--help' to see a "
                                 "brief example on the layout of this file.", default=None, required=False)
        parser.add_argument("-a", "--add-to-command-line", dest="add_to_command_line",
                            help="Adds content to the command line", default="", required=False)
        parser.add_argument("-d", "--datatypes-destination", dest="data_types_destination",
                            help="Specify the location of a datatypes_conf.xml to modify and add the registered "
                                 "data types. If the provided destination does not exist, a new file will be created.",
                            default=None, required=False)
        parser.add_argument("-x", "--default-executable-path", dest="default_executable_path",
                            help="Use this executable path when <executablePath> is not present in the CTD",
                            default=None, required=False)
        parser.add_argument("-b", "--blacklist-parameters", dest="blacklisted_parameters", default=[], nargs="+", action="append",
                            help="List of parameters that will be ignored and won't appear on the galaxy stub",
                            required=False)
        parser.add_argument("-c", "--default-category", dest="default_category", default="DEFAULT", required=False,
                            help="Default category to use for tools lacking a category when generating tool_conf.xml")
        parser.add_argument("-t", "--tool-conf-destination", dest="tool_conf_destination", default=None, required=False,
                            help="Specify the location of an existing tool_conf.xml that will be modified to include "
                                 "the converted tools. If the provided destination does not exist, a new file will"
                                 "be created.")
        parser.add_argument("-g", "--galaxy-tool-path", dest="galaxy_tool_path", default=None, required=False,
                            help="The path that will be prepended to the file names when generating tool_conf.xml")
        parser.add_argument("-r", "--required-tools", dest="required_tools_file", default=None, required=False,
                            help="Each line of the file will be interpreted as a tool name that needs translation. "
                                 "Run with '-h' or '--help' to see a brief example on the format of this file.")
        parser.add_argument("-s", "--skip-tools", dest="skip_tools_file", default=None, required=False,
                            help="File containing a list of tools for which a Galaxy stub will not be generated. "
                                 "Run with '-h' or '--help' to see a brief example on the format of this file.")
        parser.add_argument("-m", "--macros", dest="macros_files", default=[], nargs="*",
                            action="append", required=None, help="Import the additional given file(s) as macros. "
                                 "The macros stdio, requirements and advanced_options are required. Please see "
                                 "macros.xml for an example of a valid macros file. Al defined macros will be imported.")
        parser.add_argument("-p", "--hardcoded-parameters", dest="hardcoded_parameters", default=None, required=False,
                            help="File containing hardcoded values for the given parameters. Run with '-h' or '--help' "
                                 "to see a brief example on the format of this file.")
        parser.add_argument("-v", "--validation-schema", dest="xsd_location", default=None, required=False,
                            help="Location of the schema to use to validate CTDs.")

        # TODO: add verbosity, maybe?
        parser.add_argument("-V", "--version", action='version', version=program_version_message)

        # Process arguments
        args = parser.parse_args()
        
        # validate and prepare the passed arguments
        validate_and_prepare_args(args)

        # extract the names of the macros and check that we have found the ones we need
        macros_to_expand = parse_macros_files(args.macros_files)

        # parse the given supported file-formats file
        supported_file_formats = parse_file_formats(args.formats_file)

        # parse the hardcoded parameters fileĀ¬
        parameter_hardcoder = parse_hardcoded_parameters(args.hardcoded_parameters)

        # parse the skip/required tools files
        skip_tools = parse_tools_list_file(args.skip_tools_file)
        required_tools = parse_tools_list_file(args.required_tools_file)
        
        #if verbose > 0:
        #    print("Verbose mode on")
        parsed_models = convert(args.input_files,
                                args.output_destination,
                                supported_file_formats=supported_file_formats,
                                default_executable_path=args.default_executable_path,
                                add_to_command_line=args.add_to_command_line,
                                blacklisted_parameters=args.blacklisted_parameters,
                                required_tools=required_tools,
                                skip_tools=skip_tools,
                                macros_file_names=args.macros_files,
                                macros_to_expand=macros_to_expand,
                                parameter_hardcoder=parameter_hardcoder,
                                xsd_location=args.xsd_location)

        #TODO: add some sort of warning if a macro that doesn't exist is to be expanded

        # it is not needed to copy the macros files, since the user has provided them

        # generation of galaxy stubs is ready... now, let's see if we need to generate a tool_conf.xml
        if args.tool_conf_destination is not None:
            generate_tool_conf(parsed_models, args.tool_conf_destination,
                               args.galaxy_tool_path, args.default_category)

        # now datatypes_conf.xml
        if args.data_types_destination is not None:
            generate_data_type_conf(supported_file_formats, args.data_types_destination)

        return 0

    except KeyboardInterrupt:
        # handle keyboard interrupt
        return 0
    except ApplicationException, e:
        error("CTD2Galaxy could not complete the requested operation.", 0)
        error("Reason: " + e.msg, 0)
        return 1
    except ModelError, e:
        error("There seems to be a problem with one of your input CTDs.", 0)
        error("Reason: " + e.msg, 0)
        return 1
    except Exception, e:
        traceback.print_exc()
        return 2


def parse_tools_list_file(tools_list_file):
    tools_list = None
    if tools_list_file is not None:
        tools_list = []
        with open(tools_list_file) as f:
            for line in f:
                if line is None or not line.strip() or line.strip().startswith("#"):
                    continue
                else:
                    tools_list.append(line.strip())

    return tools_list


def parse_macros_files(macros_file_names):
    macros_to_expand = set()

    for macros_file_name in macros_file_names:
        try:
            macros_file = open(macros_file_name)
            info("Loading macros from %s" % macros_file_name, 0)
            root = parse(macros_file).getroot()
            for xml_element in root.findall("xml"):
                name = xml_element.attrib["name"]
                if name in macros_to_expand:
                    warning("Macro %s has already been found. Duplicate found in file %s." %
                            (name, macros_file_name), 0)
                else:
                    info("Macro %s found" % name, 1)
                    macros_to_expand.add(name)
        except ParseError, e:
            raise ApplicationException("The macros file " + macros_file_name + " could not be parsed. Cause: " +
                                       str(e))
        except IOError, e:
            raise ApplicationException("The macros file " + macros_file_name + " could not be opened. Cause: " +
                                       str(e))

    # we depend on "stdio", "requirements" and "advanced_options" to exist on all the given macros files
    missing_needed_macros = []
    for required_macro in REQUIRED_MACROS:
        if required_macro not in macros_to_expand:
            missing_needed_macros.append(required_macro)

    if missing_needed_macros:
        raise ApplicationException(
            "The following required macro(s) were not found in any of the given macros files: %s, "
            "see sample_files/macros.xml for an example of a valid macros file."
            % ", ".join(missing_needed_macros))

    # we do not need to "expand" the advanced_options macro
    macros_to_expand.remove(ADVANCED_OPTIONS_MACRO_NAME)
    return macros_to_expand


def parse_hardcoded_parameters(hardcoded_parameters_file):
    parameter_hardcoder = ParameterHardcoder()
    if hardcoded_parameters_file is not None:
        line_number = 0
        with open(hardcoded_parameters_file) as f:
            for line in f:
                line_number += 1
                if line is None or not line.strip() or line.strip().startswith("#"):
                    pass
                else:
                    # the third column must not be obtained as a whole, and not split
                    parsed_hardcoded_parameter = line.strip().split(None, 2)
                    # valid lines contain two or three columns
                    if len(parsed_hardcoded_parameter) != 2 and len(parsed_hardcoded_parameter) != 3:
                        warning("Invalid line at line number %d of the given hardcoded parameters file. Line will be"
                                "ignored:\n%s" % (line_number, line), 0)
                        continue

                    parameter_name = parsed_hardcoded_parameter[0]
                    hardcoded_value = parsed_hardcoded_parameter[1]
                    tool_names = None
                    if len(parsed_hardcoded_parameter) == 3:
                        tool_names = parsed_hardcoded_parameter[2].split(',')
                    if tool_names:
                        for tool_name in tool_names:
                            parameter_hardcoder.register_parameter(parameter_name, hardcoded_value, tool_name.strip())
                    else:
                        parameter_hardcoder.register_parameter(parameter_name, hardcoded_value)

    return parameter_hardcoder


def parse_file_formats(formats_file):
    supported_formats = {}
    if formats_file is not None:
        line_number = 0
        with open(formats_file) as f:
            for line in f:
                line_number += 1
                if line is None or not line.strip() or line.strip().startswith("#"):
                    # ignore (it'd be weird to have something like:
                    # if line is not None and not (not line.strip()) ...
                    pass
                else:
                    # not an empty line, no comment
                    # strip the line and split by whitespace
                    parsed_formats = line.strip().split()
                    # valid lines contain either one or four columns
                    if not (len(parsed_formats) == 1 or len(parsed_formats) == 3 or len(parsed_formats) == 4):
                        warning("Invalid line at line number %d of the given formats file. Line will be ignored:\n%s" %
                                (line_number, line), 0)
                        # ignore the line
                        continue
                    elif len(parsed_formats) == 1:
                        supported_formats[parsed_formats[0]] = DataType(parsed_formats[0], parsed_formats[0])
                    else:
                        mimetype = None
                        # check if mimetype was provided
                        if len(parsed_formats) == 4:
                            mimetype = parsed_formats[3]
                        supported_formats[parsed_formats[0]] = DataType(parsed_formats[0], parsed_formats[1],
                                                                        parsed_formats[2], mimetype)
    return supported_formats


def validate_and_prepare_args(args):
    # check that only one of skip_tools_file and required_tools_file has been provided
    if args.skip_tools_file is not None and args.required_tools_file is not None:
        raise ApplicationException(
            "You have provided both a file with tools to ignore and a file with required tools.\n"
            "Only one of -s/--skip-tools, -r/--required-tools can be provided.")

    # first, we convert all list of lists in args to flat lists
    lists_to_flatten = ["input_files", "blacklisted_parameters", "macros_files"]
    for list_to_flatten in lists_to_flatten:
        setattr(args, list_to_flatten, [item for sub_list in getattr(args, list_to_flatten) for item in sub_list])

    # if input is a single file, we expect output to be a file (and not a dir that already exists)
    if len(args.input_files) == 1:
        if os.path.isdir(args.output_destination):
            raise ApplicationException("If a single input file is provided, output (%s) is expected to be a file "
                                       "and not a folder.\n" % args.output_destination)
        
    # if input is a list of files, we expect output to be a folder
    if len(args.input_files) > 1:
        if not os.path.isdir(args.output_destination):
            raise ApplicationException("If several input files are provided, output (%s) is expected to be an "
                                       "existing directory.\n" % args.output_destination)

    # check that the provided input files, if provided, contain a valid file path
    input_variables_to_check = ["skip_tools_file", "required_tools_file", "macros_files", "xsd_location",
                                "input_files", "formats_file", "hardcoded_parameters"]

    for variable_name in input_variables_to_check:
        paths_to_check = []
        # check if we are handling a single file or a list of files
        member_value = getattr(args, variable_name)
        if member_value is not None:
            if isinstance(member_value, list):
                for file_name in member_value:
                    paths_to_check.append(strip(str(file_name)))
            else:
                paths_to_check.append(strip(str(member_value)))

            for path_to_check in paths_to_check:
                if not os.path.isfile(path_to_check) or not os.path.exists(path_to_check):
                    raise ApplicationException(
                        "The provided input file (%s) does not exist or is not a valid file path."
                        % path_to_check)

    # check that the provided output files, if provided, contain a valid file path (i.e., not a folder)
    output_variables_to_check = ["data_types_destination", "tool_conf_destination"]

    for variable_name in output_variables_to_check:
        file_name = getattr(args, variable_name)
        if file_name is not None and os.path.isdir(file_name):
            raise ApplicationException("The provided output file name (%s) points to a directory." % file_name)

    if not args.macros_files:
        # list is empty, provide the default value
        warning("Using default macros from macros.xml", 0)
        args.macros_files = ["macros.xml"]


def convert(input_files, output_destination, **kwargs):
    # first, generate a model
    is_converting_multiple_ctds = len(input_files) > 1 
    parsed_models = []
    schema = None
    if kwargs["xsd_location"] is not None:
        try:
            info("Loading validation schema from %s" % kwargs["xsd_location"], 0)
            schema = etree.XMLSchema(etree.parse(kwargs["xsd_location"]))
        except Exception, e:
            error("Could not load validation schema %s. Reason: %s" % (kwargs["xsd_location"], str(e)), 0)
    else:
        info("Validation against a schema has not been enabled.", 0)
    for input_file in input_files:
        try:
            if schema is not None:
                validate_against_schema(input_file, schema)
            model = CTDModel(from_file=input_file)
        except Exception, e:
            error(str(e), 1)
            continue

        if kwargs["skip_tools"] is not None and model.name in kwargs["skip_tools"]:
            info("Skipping tool %s" % model.name, 0)
            continue
        elif kwargs["required_tools"] is not None and model.name not in kwargs["required_tools"]:
            info("Tool %s is not required, skipping it" % model.name, 0)
            continue
        else:
            info("Converting from %s " % input_file, 0)
            tool = create_tool(model)
            write_header(tool, model)
            create_description(tool, model)
            expand_macros(tool, model, **kwargs)
            create_command(tool, model, **kwargs)
            create_inputs(tool, model, **kwargs)
            create_outputs(tool, model, **kwargs)
            create_help(tool, model)

            # finally, serialize the tool
            output_file = output_destination
            # if multiple inputs are being converted,
            # then we need to generate a different output_file for each input
            if is_converting_multiple_ctds:
                output_file = os.path.join(output_file, get_filename_without_suffix(input_file) + ".xml")
            # wrap our tool element into a tree to be able to serialize it
            tree = ElementTree(tool)
            tree.write(open(output_file, 'w'), encoding="UTF-8", xml_declaration=True, pretty_print=True)
            # let's use model to hold the name of the output file
            parsed_models.append([model, get_filename(output_file)])

    return parsed_models


# validates a ctd file against the schema
def validate_against_schema(ctd_file, schema):
    try:
        parser = etree.XMLParser(schema=schema)
        etree.parse(ctd_file, parser=parser)
    except etree.XMLSyntaxError, e:
        raise ApplicationException("Input ctd file %s is not valid. Reason: %s" % (ctd_file, str(e)))


def write_header(tool, model):
    tool.addprevious(etree.Comment(
        "This is a configuration file for the integration of a tools into Galaxy (https://galaxyproject.org/). "
        "This file was automatically generated using CTD2Galaxy."))
    tool.addprevious(etree.Comment('Proposed Tool Section: [%s]' % model.opt_attribs.get("category", "")))


def generate_tool_conf(parsed_models, tool_conf_destination, galaxy_tool_path, default_category):
    # for each category, we keep a list of models corresponding to it
    categories_to_tools = dict()
    for model in parsed_models:
        category = strip(model[0].opt_attribs.get("category", ""))
        if not category.strip():
            category = default_category
        if category not in categories_to_tools:
            categories_to_tools[category] = []
        categories_to_tools[category].append(model[1])
                
    # at this point, we should have a map for all categories->tools
    toolbox_node = Element("toolbox")
    
    if galaxy_tool_path is not None and not galaxy_tool_path.strip().endswith("/"):
        galaxy_tool_path = galaxy_tool_path.strip() + "/"
    if galaxy_tool_path is None:
        galaxy_tool_path = ""
    
    for category, file_names in categories_to_tools.iteritems():
        section_node = add_child_node(toolbox_node, "section")
        section_node.attrib["id"] = "section-id-" + "".join(category.split())
        section_node.attrib["name"] = category
    
        for filename in file_names:
            tool_node = add_child_node(section_node, "tool")
            tool_node.attrib["file"] = galaxy_tool_path + filename

    toolconf_tree = ElementTree(toolbox_node)
    toolconf_tree.write(open(tool_conf_destination,'w'), encoding="UTF-8", xml_declaration=True, pretty_print=True)
    info("Generated Galaxy tool_conf.xml in %s" % tool_conf_destination, 0)


def generate_data_type_conf(supported_file_formats, data_types_destination):
    data_types_node = Element("datatypes")
    registration_node = add_child_node(data_types_node, "registration")
    registration_node.attrib["converters_path"] = "lib/galaxy/datatypes/converters"
    registration_node.attrib["display_path"] = "display_applications"

    for format_name in supported_file_formats:
        data_type = supported_file_formats[format_name]
        # add only if it's a data type that does not exist in Galaxy
        if data_type.galaxy_type is not None:
            data_type_node = add_child_node(registration_node, "datatype")
            # we know galaxy_extension is not None
            data_type_node.attrib["extension"] = data_type.galaxy_extension
            data_type_node.attrib["type"] = data_type.galaxy_type
            if data_type.mimetype is not None:
                data_type_node.attrib["mimetype"] = data_type.mimetype

    data_types_tree = ElementTree(data_types_node)
    data_types_tree.write(open(data_types_destination,'w'), encoding="UTF-8", xml_declaration=True, pretty_print=True)
    info("Generated Galaxy datatypes_conf.xml in %s" % data_types_destination, 0)


# taken from
# http://stackoverflow.com/questions/8384737/python-extract-file-name-from-path-no-matter-what-the-os-path-format
def get_filename(path):
    head, tail = ntpath.split(path)
    return tail or ntpath.basename(head)


def get_filename_without_suffix(path):
    root, ext = os.path.splitext(os.path.basename(path))
    return root 


def create_tool(model):
    return Element("tool", OrderedDict([("id", model.name), ("name", model.name), ("version", model.version)]))


def create_description(tool, model):
    if "description" in model.opt_attribs.keys() and model.opt_attribs["description"] is not None:
        description = SubElement(tool,"description")
        description.text = model.opt_attribs["description"]


def get_param_cli_name(param, model):
    # we generate parameters with colons for subgroups, but not for the two topmost parents (OpenMS legacy)
    if type(param.parent) == ParameterGroup:
        if not hasattr(param.parent.parent, 'parent'):
            return resolve_param_mapping(param, model)
        elif not hasattr(param.parent.parent.parent, 'parent'):
            return resolve_param_mapping(param, model)
        else:
            if model.cli:
                warning("Using nested parameter sections (NODE elements) is not compatible with <cli>", py1)
            return get_param_name(param.parent) + ":" + resolve_param_mapping(param, model)
    else:
        return resolve_param_mapping(param, model)


def get_param_name(param):
    # we generate parameters with colons for subgroups, but not for the two topmost parents (OpenMS legacy)
    if type(param.parent) == ParameterGroup:
        if not hasattr(param.parent.parent, 'parent'):
            return param.name
        elif not hasattr(param.parent.parent.parent, 'parent'):
            return param.name
        else:
            return get_param_name(param.parent) + ":" + param.name
    else:
    	return param.name


# some parameters are mapped to command line options, this method helps resolve those mappings, if any
def resolve_param_mapping(param, model):
    # go through all mappings and find if the given param appears as a reference name in a mapping element
    param_mapping = None
    for cli_element in model.cli:
        for mapping_element in cli_element.mappings:
            if mapping_element.reference_name == param.name:
                if param_mapping is not None:
                    warning("The parameter %s has more than one mapping in the <cli> section. "
                            "The first found mapping, %s, will be used." % (param.name, param_mapping), 1)
                else:
                    param_mapping = cli_element.option_identifier

    return param_mapping if param_mapping is not None else param.name

def create_command(tool, model, **kwargs):
    final_command = get_tool_executable_path(model, kwargs["default_executable_path"]) + '\n'
    final_command += kwargs["add_to_command_line"] + '\n'
    advanced_command_start = "#if $adv_opts.adv_opts_selector=='advanced':\n"
    advanced_command_end = '#end if'
    advanced_command = ''
    parameter_hardcoder = kwargs["parameter_hardcoder"]

    found_output_parameter = False
    for param in extract_parameters(model):
        if param.type is _OutFile:
            found_output_parameter = True
        command = ''
        param_name = get_param_name(param)
        param_cli_name = get_param_cli_name(param, model)
        if param_name == param_cli_name:
            # there was no mapping, so for the cli name we will use a '-' in the prefix
            param_cli_name = '-' + param_name

        if param.name in kwargs["blacklisted_parameters"]:
            continue

        hardcoded_value = parameter_hardcoder.get_hardcoded_value(param_name, model.name)
        if hardcoded_value:
            command += '%s %s\n' % (param_cli_name, hardcoded_value)
        else:
            # parameter is neither blacklisted nor hardcoded...
            galaxy_parameter_name = get_galaxy_parameter_name(param)
            repeat_galaxy_parameter_name = get_repeat_galaxy_parameter_name(param)

            # logic for ITEMLISTs
            if param.is_list:
                if param.type is _InFile:
                    command += param_cli_name + "\n"
                    command += "  #for token in $" + galaxy_parameter_name + ":\n" 
                    command += "    $token\n"
                    command += "  #end for\n" 
                else:
                    command += "\n#if $" + repeat_galaxy_parameter_name + ":\n"
                    command += param_cli_name + "\n"
                    command += "  #for token in $" + repeat_galaxy_parameter_name + ":\n" 
                    command += "    #if \" \" in str(token):\n"
                    command += "      \"$token." + galaxy_parameter_name + "\"\n"
                    command += "    #else\n"
                    command += "      $token." + galaxy_parameter_name + "\n"
                    command += "    #end if\n"
                    command += "  #end for\n" 
                    command += "#end if\n" 
            # logic for other ITEMs 
            else:
                if param.advanced and param.type is not _OutFile:
                    actual_parameter = "$adv_opts.%s" % galaxy_parameter_name
                else:
                    actual_parameter = "$%s" % galaxy_parameter_name
                ## if whitespace_validation has been set, we need to generate, for each parameter:
                ## #if str( $t ).split() != '':
                ## -t "$t"
                ## #end if
                ## TODO only useful for text fields, integers or floats
                ## not useful for choices, input fields ...

                if not is_boolean_parameter(param) and type(param.restrictions) is _Choices :
                    command += "#if " + actual_parameter + ":\n"
                    command += '  %s\n' % param_cli_name
                    command += "  #if \" \" in str(" + actual_parameter + "):\n"
                    command += "    \"" + actual_parameter + "\"\n"
                    command += "  #else\n"
                    command += "    " + actual_parameter + "\n"
                    command += "  #end if\n"
                    command += "#end if\n" 
                elif is_boolean_parameter(param):
                    command += "#if " + actual_parameter + ":\n"
                    command += '  %s\n' % param_cli_name
                    command += "#end if\n" 
                elif TYPE_TO_GALAXY_TYPE[param.type] is 'text':
                    command += "#if " + actual_parameter + ":\n"
                    command += "  %s " % param_cli_name
                    command += "    \"" + actual_parameter + "\"\n"
                    command += "#end if\n" 
                else:
                    command += "#if " + actual_parameter + ":\n"
                    command += '  %s ' % param_cli_name
                    command += actual_parameter + "\n"
                    command += "#end if\n" 

        if param.advanced and param.type is not _OutFile:
            advanced_command += "    %s" % command
        else:
            final_command += command

    if advanced_command:
        final_command += "%s%s%s\n" % (advanced_command_start, advanced_command, advanced_command_end)

    if not found_output_parameter:
        final_command += "> $param_stdout\n" 

    command_node = add_child_node(tool, "command")
    command_node.text = final_command


# creates the xml elements needed to import the needed macros files
# and to "expand" the macros
def expand_macros(tool, model, **kwargs):
    macros_node = add_child_node(tool, "macros")
    token_node = add_child_node(macros_node, "token")
    token_node.attrib["name"] = "@EXECUTABLE@"
    token_node.text = get_tool_executable_path(model, kwargs["default_executable_path"])

    # add <import> nodes
    for macro_file_name in kwargs["macros_file_names"]:
        macro_file = open(macro_file_name)
        import_node = add_child_node(macros_node, "import")
        # do not add the path of the file, rather, just its basename
        import_node.text = os.path.basename(macro_file.name)

    # add <expand> nodes
    for expand_macro in kwargs["macros_to_expand"]:
        expand_node = add_child_node(tool, "expand")
        expand_node.attrib["macro"] = expand_macro


def get_tool_executable_path(model, default_executable_path):
    # rules to build the galaxy executable path:
    # if executablePath is null, then use default_executable_path and store it in executablePath
    # if executablePath is null and executableName is null, then the name of the tool will be used
    # if executablePath is null and executableName is not null, then executableName will be used
    # if executablePath is not null and executableName is null,
    #   then executablePath and the name of the tool will be used
    # if executablePath is not null and executableName is not null, then both will be used

    # first, check if the model has executablePath / executableName defined
    executable_path = model.opt_attribs.get("executablePath", None)
    executable_name = model.opt_attribs.get("executableName", None)

    # check if we need to use the default_executable_path
    if executable_path is None:
        executable_path = default_executable_path

    # fix the executablePath to make sure that there is a '/' in the end
    if executable_path is not None:
        executable_path = executable_path.strip()
        if not executable_path.endswith('/'):
            executable_path += '/'

    # assume that we have all information present
    command = str(executable_path) + str(executable_name)
    if executable_path is None:
        if executable_name is None:
            command = model.name
        else:
            command = executable_name
    else:
        if executable_name is None:
            command = executable_path + model.name
    return command


def get_galaxy_parameter_name(param):
    return "param_%s" % get_param_name(param).replace(':', '_').replace('-', '_')


def get_input_with_same_restrictions(out_param, model, supported_file_formats):
    for param in extract_parameters(model):
        if param.type is _InFile:
            if param.restrictions is not None:
                in_param_formats = get_supported_file_types(param.restrictions.formats, supported_file_formats)
                out_param_formats = get_supported_file_types(out_param.restrictions.formats, supported_file_formats)
                if in_param_formats == out_param_formats:
                    return param
                    

def create_inputs(tool, model, **kwargs):
    inputs_node = SubElement(tool, "inputs")

    # some suites (such as OpenMS) need some advanced options when handling inputs
    expand_advanced_node = add_child_node(tool, "expand", OrderedDict([("macro", ADVANCED_OPTIONS_MACRO_NAME)]))
    parameter_hardcoder = kwargs["parameter_hardcoder"]

    # treat all non output-file parameters as inputs
    for param in extract_parameters(model):
        # no need to show hardcoded parameters
        hardcoded_value = parameter_hardcoder.get_hardcoded_value(param.name, model.name)
        if param.name in kwargs["blacklisted_parameters"] or hardcoded_value:
            # let's not use an extra level of indentation and use NOP
            continue
        if param.type is not _OutFile:
            if param.advanced:
                if expand_advanced_node is not None:
                    parent_node = expand_advanced_node
                else:
                    # something went wrong... we are handling an advanced parameter and the
                    # advanced input macro was not set... inform the user about it
                    info("The parameter %s has been set as advanced, but advanced_input_macro has "
                         "not been set." % param.name, 1)
                    # there is not much we can do, other than use the inputs_node as a parent node!
                    parent_node = inputs_node
            else:
                parent_node = inputs_node

            # for lists we need a repeat tag
            if param.is_list and param.type is not _InFile:
                rep_node = add_child_node(parent_node, "repeat")
                create_repeat_attribute_list(rep_node, param)
                parent_node = rep_node

            param_node = add_child_node(parent_node, "param")
            create_param_attribute_list(param_node, param, kwargs["supported_file_formats"])

    # advanced parameter selection should be at the end
    # and only available if an advanced parameter exists
    if expand_advanced_node is not None and len(expand_advanced_node) > 0:
        inputs_node.append(expand_advanced_node)


def get_repeat_galaxy_parameter_name(param):
    return "rep_" + get_galaxy_parameter_name(param)


def create_repeat_attribute_list(rep_node, param):
    rep_node.attrib["name"] = get_repeat_galaxy_parameter_name(param)
    if param.required:
        rep_node.attrib["min"] = "1"
    else:
        rep_node.attrib["min"] = "0"
    # for the ITEMLISTs which have LISTITEM children we only
    # need one parameter as it is given as a string
    if param.default is not None: 
        rep_node.attrib["max"] = "1"
    rep_node.attrib["title"] = get_galaxy_parameter_name(param)


def create_param_attribute_list(param_node, param, supported_file_formats):
    param_node.attrib["name"] = get_galaxy_parameter_name(param)

    param_type = TYPE_TO_GALAXY_TYPE[param.type]
    if param_type is None:
        raise ModelError("Unrecognized parameter type %(type)s for parameter %(name)s"
                         % {"type": param.type, "name": param.name})

    if param.is_list:
        param_type = "text"

    if is_selection_parameter(param):
        param_type = "select"
        if len(param.restrictions.choices) < 5:
            param_node.attrib["display"] = "radio"
        
    if is_boolean_parameter(param):
        param_type = "boolean"
        
    if param.type is _InFile:
        # assume it's just text unless restrictions are provided
        param_format = "txt"
        if param.restrictions is not None:
            # join all formats of the file, take mapping from supported_file if available for an entry
            if type(param.restrictions) is _FileFormat:
                param_format = ','.join([get_supported_file_type(i, supported_file_formats) if
                                            get_supported_file_type(i, supported_file_formats)
                                            else i for i in param.restrictions.formats])
            else:
                raise InvalidModelException("Expected 'file type' restrictions for input file [%(name)s], "
                                            "but instead got [%(type)s]"
                                            % {"name": param.name, "type": type(param.restrictions)})

        param_node.attrib["type"] = "data"
        param_node.attrib["format"] = param_format 
        # in the case of multiple input set multiple flag
        if param.is_list:
            param_node.attrib["multiple"] = "true"

    else:
        param_node.attrib["type"] = param_type

    # check for parameters with restricted values (which will correspond to a "select" in galaxy)
    if param.restrictions is not None:
        # it could be either _Choices or _NumericRange, with special case for boolean types
        if param_type == "boolean":
            create_boolean_parameter(param_node, param)
        elif type(param.restrictions) is _Choices:
            # create as many <option> elements as restriction values
            for choice in param.restrictions.choices:
                option_node = add_child_node(param_node, "option", OrderedDict([("value", str(choice))]))
                option_node.text = str(choice)

                # preselect the default value
                if param.default == choice:
                    option_node.attrib["selected"] = "true"

        elif type(param.restrictions) is _NumericRange:
            if param.type is not int and param.type is not float:
                raise InvalidModelException("Expected either 'int' or 'float' in the numeric range restriction for "
                                            "parameter [%(name)s], but instead got [%(type)s]" %
                                            {"name": param.name, "type": type(param.restrictions)})
            # extract the min and max values and add them as attributes
            # validate the provided min and max values
            if param.restrictions.n_min is not None:
                param_node.attrib["min"] = str(param.restrictions.n_min)
            if param.restrictions.n_max is not None:
                param_node.attrib["max"] = str(param.restrictions.n_max)
        elif type(param.restrictions) is _FileFormat:
            param_node.attrib["format"] = ','.join([get_supported_file_type(i, supported_file_formats) if
                                            get_supported_file_type(i, supported_file_formats)
                                            else i for i in param.restrictions.formats])
        else:
            raise InvalidModelException("Unrecognized restriction type [%(type)s] for parameter [%(name)s]"
                                        % {"type": type(param.restrictions), "name": param.name})

        if param_type == "select" and param.default in param.restrictions.choices:
             param_node.attrib["optional"] = "False"
        else:
            param_node.attrib["optional"] = str(not param.required)

    if param_type == "text":
        # add size attribute... this is the length of a textbox field in Galaxy (it could also be 15x2, for instance)
        param_node.attrib["size"] = "30"
        # add sanitizer nodes, this is needed for special character like "["
        # which are used for example by FeatureFinderMultiplex
        sanitizer_node = SubElement(param_node, "sanitizer")

        valid_node = SubElement(sanitizer_node, "valid", OrderedDict([("initial", "string.printable")]))
        add_child_node(valid_node, "remove", OrderedDict([("value", '\'')]))
        add_child_node(valid_node, "remove", OrderedDict([("value", '"')]))

    # check for default value
    if param.default is not None and param.default is not _Null:
        if type(param.default) is list:
            # we ASSUME that a list of parameters looks like:
            # $ tool -ignore He Ar Xe
            # meaning, that, for example, Helium, Argon and Xenon will be ignored
            param_node.attrib["value"] = ' '.join(map(str, param.default))

        elif param_type != "boolean":
            param_node.attrib["value"] = str(param.default)

        else:
            # simple boolean with a default
            if param.default is True:
                param_node.attrib["checked"] = "true"
    else:
        if param.type is int or param.type is float:
            # galaxy requires "value" to be included for int/float
            # since no default was included, we need to figure out one in a clever way... but let the user know
            # that we are "thinking" for him/her
            warning("Generating default value for parameter [%s]. "
                    "Galaxy requires the attribute 'value' to be set for integer/floats. "
                    "Edit the CTD file and provide a suitable default value." % param.name, 1)
            # check if there's a min/max and try to use them
            default_value = None
            if param.restrictions is not None:
                if type(param.restrictions) is _NumericRange:
                    default_value = param.restrictions.n_min
                    if default_value is None:
                        default_value = param.restrictions.n_max
                    if default_value is None:
                        # no min/max provided... just use 0 and see what happens
                        default_value = 0
                else:
                    # should never be here, since we have validated this anyway...
                    # this code is here just for documentation purposes
                    # however, better safe than sorry!
                    # (it could be that the code changes and then we have an ugly scenario)
                    raise InvalidModelException("Expected either a numeric range for parameter [%(name)s], "
                                                "but instead got [%(type)s]"
                                                % {"name": param.name, "type": type(param.restrictions)})
            else:
                # no restrictions and no default value provided...
                # make up something
                default_value = 0
            param_node.attrib["value"] = str(default_value)

    label = "%s parameter" % param.name
    help_text = ""

    if param.description is not None:
        label, help_text = generate_label_and_help(param.description)

    param_node.attrib["label"] = label
    param_node.attrib["help"] = "(-%s)" % param.name + " " + help_text


def generate_label_and_help(desc):
    label = ""
    help_text = ""
    # This tag is found in some descriptions 
    if not isinstance(desc, basestring):
        desc = str(desc)
    desc = desc.encode("utf8").replace("#br#", " <br>")
    # Get rid of dots in the end
    if desc.endswith("."):
       desc = desc.rstrip(".")
    # Check if first word is a normal word and make it uppercase
    if str(desc).find(" ") > -1:
        first_word, rest = str(desc).split(" ", 1)
        if str(first_word).islower():
            # check if label has a quotient of the form a/b
            if first_word.find("/") != 1 :
                first_word.capitalize()
        desc = first_word + " " + rest
    label = desc.decode("utf8")
    
    # Try to split the label if it is too long    
    if len(desc) > 50:
        # find an example and put everything before in the label and the e.g. in the help
        if desc.find("e.g.") > 1 :
            label, help_text = desc.split("e.g.",1)
            help_text = "e.g." + help_text
        else:
            # find the end of the first sentence
            # look for ". " because some labels contain .file or something similar
            delimiter = ""
            if desc.find(". ") > 1 and desc.find("? ") > 1:
                if desc.find(". ") < desc.find("? "):
                    delimiter = ". "
                else:
                    delimiter = "? "
            elif desc.find(". ") > 1:
                delimiter = ". "
            elif desc.find("? ") > 1:
                delimiter = "? "
            if delimiter != "":
                label, help_text = desc.split(delimiter, 1)

            # add the question mark back
            if delimiter == "? ":
                label += "? "
    
    # remove all linebreaks
    label = label.rstrip().rstrip('<br>').rstrip()
    return label, help_text


def get_indented_text(text, indentation_level):
    return ("%(indentation)s%(text)s" %
            {"indentation": "  " * (MESSAGE_INDENTATION_INCREMENT * indentation_level),
             "text": text})


def warning(warning_text, indentation_level):
    sys.stdout.write(get_indented_text("WARNING: %s\n" % warning_text, indentation_level))


def error(error_text, indentation_level):
    sys.stderr.write(get_indented_text("ERROR: %s\n" % error_text, indentation_level))


def info(info_text, indentation_level):
    sys.stdout.write(get_indented_text("INFO: %s\n" % info_text, indentation_level))


# determines if the given choices are boolean (basically, if the possible values are yes/no, true/false)
def is_boolean_parameter(param):
    ## detect boolean selects of OpenMS
    if is_selection_parameter(param):
        if len(param.restrictions.choices) == 2:
            # check that default value is false to make sure it is an actual flag
            if "false" in param.restrictions.choices and \
                "true" in param.restrictions.choices and \
                param.default == "false":
                return True
    else:
        return param.type is bool


# determines if there are choices for the parameter
def is_selection_parameter(param):
    return type(param.restrictions) is _Choices


def get_lowercase_list(some_list):
    lowercase_list = map(str, some_list)
    lowercase_list = map(string.lower, lowercase_list)
    lowercase_list = map(strip, lowercase_list)
    return lowercase_list


# creates a galaxy boolean parameter type
# this method assumes that param has restrictions, and that only two restictions are present
# (either yes/no or true/false)
def create_boolean_parameter(param_node, param):
    # first, determine the 'truevalue' and the 'falsevalue'
    """TODO: true and false values can be way more than 'true' and 'false'
        but for that we need CTD support
    """
    # by default, 'true' and 'false' are handled as flags, like the verbose flag (i.e., -v)
    true_value = "-%s" % get_param_name(param)
    false_value = ""
    choices = get_lowercase_list(param.restrictions.choices)
    if "yes" in choices:
        true_value = "yes"
        false_value = "no"
    param_node.attrib["truevalue"] = true_value
    param_node.attrib["falsevalue"] = false_value

    # set the checked attribute
    if param.default is not None:
        checked_value = "false"
        default = strip(string.lower(param.default))
        if default == "yes" or default == "true":
            checked_value = "true"
        #attribute_list["checked"] = checked_value
        param_node.attrib["checked"] = checked_value


def create_outputs(parent, model, **kwargs):
    outputs_node = add_child_node(parent, "outputs")
    parameter_hardcoder = kwargs["parameter_hardcoder"]

    for param in extract_parameters(model):

        # no need to show hardcoded parameters
        hardcoded_value = parameter_hardcoder.get_hardcoded_value(param.name, model.name)
        if param.name in kwargs["blacklisted_parameters"] or hardcoded_value:
            # let's not use an extra level of indentation and use NOP
            continue
        if param.type is _OutFile:
            create_output_node(outputs_node, param, model, kwargs["supported_file_formats"])

    # If there are no outputs defined in the ctd the node will have no children
    # and the stdout will be used as output
    if len(outputs_node) == 0:
        add_child_node(outputs_node, "data",
                       OrderedDict([("name", "param_stdout"), ("format", "txt"), ("label", "Output from stdout")]))


def create_output_node(parent, param, model, supported_file_formats):
    data_node = add_child_node(parent, "data")
    data_node.attrib["name"] = get_galaxy_parameter_name(param)

    data_format = "data"
    if param.restrictions is not None:
        if type(param.restrictions) is _FileFormat:
            # set the first data output node to the first file format

            # check if there are formats that have not been registered yet...
            output = list()
            for format_name in param.restrictions.formats:
                if not format_name in supported_file_formats.keys():
                    output.append(str(format_name))

            # warn only if there's about to complain
            if output:
                warning("Parameter " + param.name + " has the following unsupported format(s):" + ','.join(output), 1)
                data_format = ','.join(output)

            formats = get_supported_file_types(param.restrictions.formats, supported_file_formats)
            try:
                data_format = formats.pop()
            except KeyError:
                # there is not much we can do, other than catching the exception
                pass
            # if there are more than one output file formats try to take the format from the input parameter
            if formats:
                corresponding_input = get_input_with_same_restrictions(param, model, supported_file_formats)
                if corresponding_input is not None:
                    data_format = "input"
                    data_node.attrib["metadata_source"] = get_galaxy_parameter_name(corresponding_input)
        else:
            raise InvalidModelException("Unrecognized restriction type [%(type)s] "
                                        "for output [%(name)s]" % {"type": type(param.restrictions),
                                                                   "name": param.name})
    data_node.attrib["format"] = data_format

    #TODO: find a smarter label ?
    #if param.description is not None:
    #    data_node.setAttribute("label", param.description)
    return data_node


# Get the supported file format for one given format
def get_supported_file_type(format_name, supported_file_formats):
    if format_name in supported_file_formats.keys():
        return supported_file_formats.get(format_name, DataType(format_name, format_name)).galaxy_extension
    else:
        return None


def get_supported_file_types(formats, supported_file_formats):
    return set([supported_file_formats.get(format_name, DataType(format_name, format_name)).galaxy_extension
               for format_name in formats if format_name in supported_file_formats.keys()])


def create_change_format_node(parent, data_formats, input_ref):
    #  <change_format>
    #    <when input="secondary_structure" value="true" format="txt"/>
    #  </change_format>
    change_format_node = add_child_node(parent, "change_format")
    for data_format in data_formats:
        add_child_node(change_format_node, "when",
                       OrderedDict([("input", input_ref), ("value", data_format), ("format", data_format)]))


# Shows basic information about the file, such as data ranges and file type.
def create_help(tool, model):
    manual = ''
    doc_url = None
    if 'manual' in model.opt_attribs.keys(): 
        manual += '%s\n\n' % model.opt_attribs["manual"]
    if 'docurl' in model.opt_attribs.keys():
        doc_url = model.opt_attribs["docurl"]

    help_text = "No help available"
    if manual is not None:
        help_text = manual
    if doc_url is not None:
        help_text = ("" if manual is None else manual) + "\nFor more information, visit %s" % doc_url
    help_node = add_child_node(tool, "help")
    # TODO: do we need CDATA Section here?
    help_node.text = help_text


# since a model might contain several ParameterGroup elements,
# we want to simply 'flatten' the parameters to generate the Galaxy wrapper
def extract_parameters(model):
    parameters = []
    if len(model.parameters.parameters) > 0:
        # use this to put parameters that are to be processed
        # we know that CTDModel has one parent ParameterGroup
        pending = [model.parameters]
        while len(pending) > 0:
            # take one element from 'pending'
            parameter = pending.pop()
            if type(parameter) is not ParameterGroup:
                parameters.append(parameter)
            else:
                # append the first-level children of this ParameterGroup
                pending.extend(parameter.parameters.values()) 
    # returned the reversed list of parameters (as it is now,
    # we have the last parameter in the CTD as first in the list)
    return reversed(parameters)


# adds and returns a child node using the given name to the given parent node
def add_child_node(parent_node, child_node_name, attributes=OrderedDict([])):
    child_node = SubElement(parent_node, child_node_name, attributes)
    return child_node


if __name__ == "__main__":
    sys.exit(main())