Automatic .txt to XML converter

git-svn-id: svn://svn.code.sf.net/p/sc2/code/trunk@209 8092fc87-c524-0410-9efc-e669fe64eaf9

tools/script/Dialogue.py

@@ -0,0 +1,116 @@
+#!/usr/bin/python
+import sys
+import string
+
+def _xml_line(line, firstline):
+    if len(line) == 0 and firstline: return ''
+    return '    <line> '+line.strip()+' </line>\n'
+
+def _xml_join(a, tag, b):
+    if tag is None:
+        return a.strip() + ' '+ b.lstrip()
+    result=a.strip()
+    if a != '' and a[-1] in string.whitespace:
+        result += " <space/>"
+    result += ' <special type="'+tag+'"/> '
+    if b != '' and b[0] in string.whitespace:
+        result += "<space/> "
+    result +=  b.lstrip()
+    return result
+
+class Speech:
+    def __init__(self, id=None, audio=None):
+        self.id = id
+        self.audio = audio
+        self.lines = []
+    def add_line (self, line):
+        if len(line) != 0 and line[-1] == '\n': line = line[:-1]
+        self.lines.append(line)
+    def clean (self):
+        while self.lines != []:
+            if self.lines[-1].strip() != "": break
+            self.lines.pop()
+    def join (self, tag, other):
+        if self.lines == []:
+            self.lines = ['']
+        if other is None:
+            self.lines[-1] = _xml_join(self.lines[-1], tag, '')
+            return
+        if other.lines == []:
+            other.lines = ['']
+        self.lines[-1] = _xml_join(self.lines[-1], tag, other.lines[0])
+        self.lines.extend(other.lines[1:])        
+
+class Speeches:
+    def __init__(self, datafile):
+        current = None
+        speeches = []
+        self.speechdict = {}
+        for line in datafile:
+            if line[0] == '#':
+                lparen = line.index('(')
+                rparen = line.rindex(')')
+                if (lparen > rparen): raise ValueError
+                id = line[(lparen+1):rparen]
+                if (lparen != 1):
+                    print "Warning: Speech "+id+" in "+filename+" isn't flush left"
+                audio = line[rparen+1:].strip()
+                if audio == "": audio = None
+                current = Speech(id, audio)
+                speeches.append(current)
+            else:
+                current.add_line(line)
+        for speech in speeches: speech.clean()
+        self.speecharray = speeches
+        self.__rehash()
+    def as_xml(self):
+        result =  '<?xml version="1.0" encoding="UTF-8"?>\n'
+        result += '<!DOCTYPE speech SYSTEM "speech.dtd">\n\n'
+        result += '<speeches>\n'
+       
+        for speech in self.speecharray:
+            result += '  <speech id="'+speech.id+'">\n'
+            if speech.audio is not None:
+                result += '    <audio><file>'+speech.audio+'</file></audio>\n'
+            for line, i in zip(speech.lines, xrange(sys.maxint)):
+                result += _xml_line(line, i==0)
+            result += '  </speech>\n'
+
+        result += '</speeches>\n'
+        return result
+    def __rehash(self):
+        self.speechdict = {}
+        for speech, i in zip(self.speecharray, xrange(sys.maxint)):
+            self.speechdict[speech.id]=i
+    def deletespeech(self, id):
+        i = self.speechdict[id]
+        del self.speecharray[i]
+        self.__rehash()
+    def renamespeech(self, oldid, newid):
+        i = self.speechdict[oldid]
+        del self.speechdict[oldid]
+        self.speechdict[newid]=i
+        self.speecharray[i].id=newid
+    def joinspeech(self, srcid, tag, nextid):
+        src = self.speecharray[self.speechdict[srcid]]
+        if nextid is not None:
+            next = self.speecharray[self.speechdict[nextid]]
+        else:
+            next = None
+        src.join(tag, next)
+    def addtospeech(self, id, str):
+        src = self.speecharray[self.speechdict[id]]
+        if src.lines == []:
+            src.lines = [str]
+        else:
+            src.lines[-1] = _xml_join(src.lines[-1], None, str)
+    def pretag(self, id, tag):
+        src = self.speecharray[self.speechdict[id]]
+        if src.lines == []:
+            src.lines = ['']
+        src.lines[0] = _xml_join('', tag, src.lines[0])
+
+
+if __name__ == '__main__':
+    print "This is the core library for the Ur-Quan Masters dialogue translators."
+    print "You don't need to run it directly."

tools/script/config.txt

@@ -0,0 +1,118 @@
+fuse   chmmr:   i_am_captain=i_am_captain0 captain i_am_captain1 ship i_am_captain2
+fuse   comandr: name_4 = name_40 captain name_41
+fuse   comandr: we_are_vindicator=we_are_vindicator0 captain we_are_vindicator1 ship we_are_vindicator2
+fuse   comandr: no_but_well_help=no_but_well_help0 ship no_but_well_help1
+fuse   melnorm: we_are_from_alliance=we_are_from_alliance0 alliance
+addstr melnorm: we_are_from_alliance !
+fuse   orz:     we_are_vindicator=we_are_vindicator0 captain we_are_vindicator1 ship we_are_vindicator2
+fuse   pkunk:   conquer_because_1=conquer_because_1 alliance
+addstr pkunk:   conquer_because_1 !
+fuse   pkunk:   we_are_vindicator=we_are_vindicator0 captain we_are_vindicator1 ship we_are_vindicator2
+fuse   shofixt: i_am_captain=i_am_captain0 captain i_am_captain1 alliance i_am_captain2 ship i_am_captain3
+fuse   shofixt: look=look0 shofixti_name look1
+fuse   shofixt: report=report0 shofixti_name report1
+fuse   shofixt: why_here=why_here0 shofixti_name why_here1
+fuse   shofixt: bye=bye0 shofixti_name bye1
+fuse   slyhome: we_are_us=we_are_us0 captain we_are_us1 ship we_are_us2
+fuse   spahome: we_are_vindicator=we_are_vindicator0 captain we_are_vindicator1 ship we_are_vindicator2
+fuse   starbas: tell_me_about_modules=tell_me_about_modules0 ship tell_me_about_modules1
+fuse   starbas: tell_me_about_fuel=tell_me_about_fuel0 ship tell_me_about_fuel1
+fuse   supox:   i_am=i_am0 captain i_am1
+fuse   supox:   my_ship=my_ship0 ship my_ship1
+fuse   supox:   from_alliance=from_alliance0 alliance from_alliance1
+fuse   syreen:  we_are_vindicator=we_are_vindicator0 captain we_are_vindicator1 ship we_are_vindicator2
+fuse   talkpet: we_are_vindicator=we_are_vindicator0 ship we_are_vindicator1
+fuse   thradd:  alliance_name=alliance_name alliance
+fuse   umgah:   we_vindicator=we_vindicator0 captain we_vindicator1 ship we_vindicator2
+fuse   utwig:   we_are_vindicator=we_are_vindicator0 captain we_are_vindicator1 alliance we_are_vindicator2
+fuse   yehat:   i_demand_you_ally_homeworld=i_demand_you_ally_homeworld0 captain i_demand_you_ally_homeworld1 alliance i_demand_you_ally_homeworld2 ship i_demand_you_ally_homeworld3
+fuse   yehat:   i_demand_you_ally_space=i_demand_you_ally_space0 captain i_demand_you_ally_space1 ship i_demand_you_ally_space2 alliance i_demand_you_ally_space3
+fuse   zoqfot:  we_are_vindicator=we_are_vindicator0 alliance_possessive we_are_vindicator1 ship we_are_vindicator2
+
+fuse   comandr: OK_THE_NAME_IS_EMPIRE=OK_THE_NAME_IS_EMPIRE0 captain OK_THE_NAME_IS_EMPIRE1
+fuse   druuge:  FUEL=FUEL0 earned_units FUEL1
+fuse   melnorm: SOLD_LIFE_DATA=SOLD_LIFE_DATA1 sell_count SOLD_LIFE_DATA2 earned_units SOLD_LIFE_DATA3
+fuse   melnorm: SOLD_RAINBOW_LOCATIONS=SOLD_RAINBOW_LOCATIONS1 sell_count SOLD_RAINBOW_LOCATIONS2 earned_units SOLD_RAINBOW_LOCATIONS3
+fuse   melnorm: CREDIT_IS=CREDIT_IS0 credits CREDIT_IS1
+fuse   melnorm: NEED_MORE_CREDIT=NEED_MORE_CREDIT0 more_credits NEED_MORE_CREDIT1
+fuse   pkunk:   NOT_CONQUER_1=NOT_CONQUER_10 alliance NOT_CONQUER_11 alliance NOT_CONQUER_12
+fuse   shofixt: GOODBYE=GOODBYE0 captain GOODBYE1
+fuse   shofixt: FRIENDLY_HELLO=FRIENDLY_HELLO0 captain FRIENDLY_HELLO1 ship FRIENDLY_HELLO2
+fuse   shofixt: NOTHING_NEW=NOTHING_NEW0 captain NOTHING_NEW1
+fuse   spahome: JUST_MISUNDERSTANDING=JUST_MISUNDERSTANDING0 captain JUST_MISUNDERSTANDING1
+fuse   starbas: NORMAL_HELLO_A=NORMAL_HELLO_A0 captain NORMAL_HELLO_A1
+fuse   starbas: NORMAL_HELLO_B=NORMAL_HELLO_B0 captain NORMAL_HELLO_B1
+fuse   starbas: NORMAL_HELLO_C=NORMAL_HELLO_C0 captain NORMAL_HELLO_C1
+fuse   starbas: NORMAL_HELLO_D=NORMAL_HELLO_D0 captain NORMAL_HELLO_D1
+fuse   starbas: NORMAL_HELLO_E=NORMAL_HELLO_E0 captain NORMAL_HELLO_E1
+fuse   starbas: NORMAL_HELLO_F=NORMAL_HELLO_F0 captain NORMAL_HELLO_F1
+fuse   starbas: NORMAL_HELLO_G=NORMAL_HELLO_G0 captain NORMAL_HELLO_G1
+fuse   starbas: NORMAL_HELLO_H=NORMAL_HELLO_H0 captain NORMAL_HELLO_H1
+fuse   starbas: NORMAL_GOODBYE_A=NORMAL_GOODBYE_A0 captain NORMAL_GOODBYE_A1
+fuse   starbas: NORMAL_GOODBYE_B=NORMAL_GOODBYE_B0 captain NORMAL_GOODBYE_B1
+fuse   starbas: NORMAL_GOODBYE_C=NORMAL_GOODBYE_C0 captain NORMAL_GOODBYE_C1
+fuse   starbas: NORMAL_GOODBYE_D=NORMAL_GOODBYE_D0 captain NORMAL_GOODBYE_D1
+fuse   starbas: NORMAL_GOODBYE_E=NORMAL_GOODBYE_E0 captain NORMAL_GOODBYE_E1
+fuse   starbas: NORMAL_GOODBYE_F=NORMAL_GOODBYE_F0 captain NORMAL_GOODBYE_F1
+fuse   starbas: NORMAL_GOODBYE_G=NORMAL_GOODBYE_G0 captain NORMAL_GOODBYE_G1
+fuse   starbas: NORMAL_GOODBYE_H=NORMAL_GOODBYE_H0 captain NORMAL_GOODBYE_H1
+fuse   starbas: LIGHT_LOAD_A=LIGHT_LOAD_A0 captain LIGHT_LOAD_A1
+fuse   starbas: LIGHT_LOAD_B=LIGHT_LOAD_B0 captain LIGHT_LOAD_B1
+fuse   starbas: LIGHT_LOAD_C=LIGHT_LOAD_C0 captain LIGHT_LOAD_C1
+fuse   starbas: LIGHT_LOAD_D=LIGHT_LOAD_D0 captain LIGHT_LOAD_D1
+fuse   starbas: LIGHT_LOAD_E=LIGHT_LOAD_E0 captain LIGHT_LOAD_E1
+fuse   starbas: LIGHT_LOAD_F=LIGHT_LOAD_F0 captain LIGHT_LOAD_F1
+fuse   starbas: LIGHT_LOAD_G=LIGHT_LOAD_G0 captain LIGHT_LOAD_G1
+fuse   starbas: MEDIUM_LOAD_A=MEDIUM_LOAD_A0 captain MEDIUM_LOAD_A1
+fuse   starbas: MEDIUM_LOAD_B=MEDIUM_LOAD_B0 captain MEDIUM_LOAD_B1
+fuse   starbas: MEDIUM_LOAD_C=MEDIUM_LOAD_C0 captain MEDIUM_LOAD_C1
+fuse   starbas: MEDIUM_LOAD_D=MEDIUM_LOAD_D0 captain MEDIUM_LOAD_D1
+fuse   starbas: MEDIUM_LOAD_E=MEDIUM_LOAD_E0 captain MEDIUM_LOAD_E1
+fuse   starbas: MEDIUM_LOAD_F=MEDIUM_LOAD_F0 captain MEDIUM_LOAD_F1
+fuse   starbas: MEDIUM_LOAD_G=MEDIUM_LOAD_G0 captain MEDIUM_LOAD_G1
+fuse   starbas: HEAVY_LOAD_A=HEAVY_LOAD_A0 captain HEAVY_LOAD_A1
+fuse   starbas: HEAVY_LOAD_B=HEAVY_LOAD_B0 captain HEAVY_LOAD_B1
+fuse   starbas: HEAVY_LOAD_C=HEAVY_LOAD_C0 captain HEAVY_LOAD_C1
+fuse   starbas: HEAVY_LOAD_D=HEAVY_LOAD_D0 captain HEAVY_LOAD_D1
+fuse   starbas: HEAVY_LOAD_E=HEAVY_LOAD_E0 captain HEAVY_LOAD_E1
+fuse   starbas: HEAVY_LOAD_F=HEAVY_LOAD_F0 captain HEAVY_LOAD_F1
+fuse   starbas: HEAVY_LOAD_G=HEAVY_LOAD_G0 captain HEAVY_LOAD_G1
+fuse   starbas: STARBASE_IS_READY=STARBASE_IS_READY_A ship STARBASE_IS_READY_B ship STARBASE_IS_READY_C
+fuse   starbas: ABOUT_CREW=ABOUT_CREW0 ship ABOUT_CREW1
+fuse   syreen:  WELCOME_VINDICATOR=WELCOME_VINDICATOR0 captain WELCOME_VINDICATOR1 ship WELCOME_VINDICATOR2
+fuse   syreen:  MORE_COMFORTABLE=MORE_COMFORTABLE0 captain MORE_COMFORTABLE1
+fuse   syreen:  THIS_HAPPENS=THIS_HAPPENS0 captain THIS_HAPPENS1
+fuse   thradd:  WELCOME_SPACE=WELCOME_SPACE0 culture_20 WELCOME_SPACE1
+fuse   thradd:  WELCOME_HOMEWORLD=WELCOME_HOMEWORLD0 culture_20 WELCOME_HOMEWORLD1
+fuse   thradd:  SLAVE_EMPIRE=SLAVE_EMPIRE captain
+
+pretag melnorm: LANDERS landers
+pretag melnorm: THRUSTERS thrusters
+pretag melnorm: JETS jets
+pretag melnorm: PODS pods
+pretag melnorm: BAYS bays
+pretag melnorm: DYNAMOS dynamos
+pretag melnorm: FURNACES furnaces
+pretag melnorm: GUNS guns
+pretag melnorm: BLASTERS blasters
+pretag melnorm: CANNONS cannons
+pretag melnorm: TRACKERS trackers
+pretag melnorm: DEFENSES defenses
+
+add    starbas: NORMAL_HELLO_A NORMAL_HELLO_TAIL
+add    starbas: NORMAL_HELLO_B NORMAL_HELLO_TAIL
+add    starbas: NORMAL_HELLO_C NORMAL_HELLO_TAIL
+add    starbas: NORMAL_HELLO_D NORMAL_HELLO_TAIL
+add    starbas: NORMAL_HELLO_E NORMAL_HELLO_TAIL
+add    starbas: NORMAL_HELLO_F NORMAL_HELLO_TAIL
+add    starbas: NORMAL_HELLO_G NORMAL_HELLO_TAIL
+add    starbas: NORMAL_HELLO_H NORMAL_HELLO_TAIL
+add    thradd:  OK_SLAVE SLAVE_EMPIRE
+add    thradd:  the_slave_empire SLAVE_EMPIRE
+
+delete melnorm: name_1 name_2 name_3 name_40 name_41
+delete pkunk:   name_1 name_2 name_3 name_40 name_41
+delete shofixt: name_1 name_2 name_3 name_40 name_41
+delete starbas: NORMAL_HELLO_TAIL
+delete supox:   name_1 name_2 name_3 name_40 name_41
+delete thradd:  name_1 name_2 name_3 name_40 name_41

tools/script/scriptcheck.py

@@ -1,4 +1,4 @@
-#!/usr/local/bin/python2.2
+#!/usr/bin/python
 
 import string
 import os.path
@@ -98,8 +98,8 @@ def read_source_header(filename):
 races = ['arilou', 'blackur', 'chmmr', 'comandr', 'druuge',
          'ilwrath', 'melnorm', 'mycon', 'orz', 'pkunk', 'rebel',
          'shofixt', 'slyhome', 'slyland', 'spahome', 'spathi',
-         'starbas', 'supox', 'syreen', 'talkpet', 'utwig',
+         'starbas', 'supox', 'syreen', 'talkpet', 'thradd',
-         'vux', 'yehat', 'zoqfot']
+         'umgah', 'urquan', 'utwig', 'vux', 'yehat', 'zoqfot']
 
 root = os.path.join('..','..','sc2')
 

tools/script/xmlize.py

@@ -0,0 +1,129 @@
+#!/usr/bin/python
+
+import sys
+import os.path
+import Dialogue
+
+races = ['arilou', 'blackur', 'chmmr', 'comandr', 'druuge',
+         'ilwrath', 'melnorm', 'mycon', 'orz', 'pkunk', 'rebel',
+         'shofixt', 'slyhome', 'slyland', 'spahome', 'spathi',
+         'starbas', 'supox', 'syreen', 'talkpet', 'thradd', 'utwig',
+         'umgah', 'urquan', 'vux', 'yehat', 'zoqfot']
+
+root = os.path.join('..','..','sc2')
+
+race_speeches = {}
+
+def load_txt():
+    print "Loading .txt files...",
+    for race in races:
+        try:
+            filename = os.path.join(root,'content','comm',race,race+'.txt')
+            speeches = Dialogue.Speeches(file(filename))
+            race_speeches[race] = speeches
+            print race,
+        except ValueError:
+            print "["+race+" didn't parse]",
+        except IOError:
+            print "["+race+" didn't load]",
+        sys.stdout.flush()
+    print ""
+
+def do_commands():
+    def do_delete(speeches, argstr):
+        args = argstr.split()
+        # Check all args before deleting any of them
+        for arg in args:
+            temp = speeches.speechdict[arg]
+        for arg in args:
+            speeches.deletespeech(arg)
+    def do_fuse(speeches, argstr):
+        i = argstr.index('=')
+        left = argstr[:i].split()
+        right = argstr[i+1:].split()
+        if len(left) != 1: raise ValueError
+        if len(right) == 0: raise ValueError
+        new_id = left[0]
+        old_id = right[0]
+        speeches.renamespeech(old_id, new_id)
+        # Split out tags from items.
+        is_tag = 1
+        tags = []
+        speechIDs = []
+        for arg in right[1:]:
+            if is_tag:
+                tags.append(arg)
+                is_tag = 0
+            else:
+                speechIDs.append(arg)
+                is_tag = 1
+        # Now do the actual join
+        for tag, speech in map(None, tags, speechIDs):
+            speeches.joinspeech(new_id, tag, speech)
+        for speech in speechIDs:
+            speeches.deletespeech(speech)
+    def do_add(speeches, argstr):
+        args = argstr.split()
+        if len(args) == 0: raise ValueError
+        for arg in args[1:]:
+            speeches.joinspeech(args[0], None, arg)
+    def do_addstr(speeches, argstr):
+        args = argstr.split()
+        if len(args) == 0: raise ValueError
+        id = args[0]
+        str = " ".join(args[1:])
+        speeches.addtospeech(id, str)
+    def do_pretag(speeches, argstr):
+        args = argstr.split()
+        if len(args) != 2: raise ValueError
+        id = args[0]
+        tag = args[1]
+        speeches.pretag(id, tag)
+    directives = {'delete':do_delete, 'fuse':do_fuse,
+                  'add':do_add, 'addstr':do_addstr,
+                  'pretag':do_pretag}
+    print "Processing command file..."
+    try:
+        cmdfile = file(sys.argv[1])
+        for cmd in cmdfile:
+            try:
+                i = cmd.index(':')
+                left = cmd[:i].split()
+                right = cmd[i+1:]
+                if len(left) != 2: raise ValueError
+                directive = directives[left[0].lower()]
+                race = race_speeches[left[1]]
+                directive(race, right)
+            except ValueError:
+                if cmd.strip() != "":
+                    print "Malformed command '%s'" % cmd.strip()
+            except KeyError:
+                print "Bad ID in '%s'" % cmd.strip()
+        cmdfile.close()
+    except IndexError:
+        print "Well, we would have if it were specified."
+    except IOError:
+        print "Well, we would have had the file specified actually existed."
+    
+def write_XML():
+    print "Writing XML...",
+    for race in races:
+        try:
+            speeches = race_speeches[race]
+            outfilename = os.path.join('.', race+'.xml');
+            race_xml = speeches.as_xml()
+            xmlfile = file(outfilename, 'wt')
+            xmlfile.write(race_xml)
+            xmlfile.close()
+            print race,
+        except KeyError:
+            print "["+race+" didn't load or process]",
+        except IOError:
+            print "["+race+" couldn't write]",
+        sys.stdout.flush()
+    print ""
+        
+if __name__ == "__main__":
+    load_txt()
+    do_commands()
+    write_XML()