"""decrunch.py

Part of the Tundra project.
(C) 2006 Rebuild Software
http://rebui.ld.net.au

This file is currently a nasty mess of pseudostructs (thanks to the beloved
Python dictionary and list) and unreadable bollocks. There's no error checking
as such, generally you trust that this program is perfect and if you get an 
exception then -obviously- the file is formatted wrong.

Currently there are methods to:
- read raw DOS Lemmings level files, convert to Level objects
- read crunched DOS Lemmings DAT files and decrunch them into raw chunks
- extract graphics/objectdata from GroundXo.dat + VgagrX.dat pairs, convert
  to XML and PNG
- extract level pixmaps from vgaspec files
- extract the important graphics from main.dat

Extra-special thanks must be communicated to Lemmingologist, ccexplore 
and Mindless, as many of the methods described in this file are based on their
extensive findings.
"""

import os, sys
from level import *

import globals as g
from excepts import *

import Image, PngImagePlugin, gzip     # used to allow indexed PNG dumps
import xml.sax.writer

# name constants for each of the graphic tilesets
styles = ["Soil","Fire","Marble","Pillar","Ice","Brick","Rock","Snow","Bubble"]
styles_onml = ["Brick","Rock","Snow","Bubble"]
difficulty = ["Fun","Tricky","Taxing","Mayhem"]
difficulty_onml = ["Tame","Crazy","Wild","Wicked","Havoc"]

ega          = [(  0,  0,  0),(  0,  0,175),(  0,175,  0),(  0,175,175),
                (175,  0,  0),(175,  0,175),(175, 80,  0),(175,175,175),
                None, None, None, None, None, None, None, None,
                ( 80, 80, 80),( 80, 80,255),( 80,255, 80),( 80,255,255),
                (255, 80, 80),(255, 80,255),(255,255, 80),(255,255,255)]
            
menu_pal     = [(  0,  0,  0),(128, 64, 32),( 96, 48, 32),( 48,  0, 16),
                ( 32,  8,124),( 64, 44,144),(104, 88,164),(152,140,188),
                (  0, 80,  0),(  0, 96, 16),(  0,112, 32),(  0,128, 64),
                (208,208,208),(176,176,  0),( 64, 80,176),(224,128,144)]

# the brick/dirt colour is set here to be brown
default_pal  = [(  0,  0,  0),( 65, 65,227),(  0,178,  0),(243,210,210),
                (243,243,  0),(243, 32, 32),(130,130,130),(210,130, 32)]

masks_pal = [(  0,  0,  0), g.DEFAULT_COLOURKEY[:3]]

numbers_pal = [(  0,  0,  0), (255,255,255)]

rgbToPrintable = lambda (r,g,b): repr((r,g,b))[1:][:-1]

def reverseBits(byte):
    breakup = []
    for i in range(0,8):
        breakup.append(byte%2)
        byte >>= 1 
    output = 0
    for i in range(-1,-9,-1):
        output += breakup[i]<<((i+1)*-1)
    return output

def getBit(state):
    if state["bitIndex"] >= 8:
        state["index"] += 1
        state["bitIndex"] = 0
        state["currBits"] = reverseBits(ord(state["source"][state["index"]]))
    bit = state["currBits"]%2
    state["currBits"] >>= 1
    state["bitIndex"] += 1
    return bit

def getFile(filename, srcpath):
    """getFile(filename) -> Returns a string of the contents from the (case 
    insensitive) filename from inside LEMMINGS_PATH"""
    filename = filename.lower()
    l = os.listdir(srcpath)
    lu = [x.lower() for x in l]
    path = srcpath + l[lu.index(filename)]
    return open(path,"rb").read()

def checkDir(srcpath):
    l = os.listdir(srcpath)
    lu = [x.lower() for x in l]
    check = lambda f: f in lu
    
    classic_levels  = ["level00"+repr(i)+".dat" for i in range(0,10)]
    classic_levels += ["oddtable.dat"]
    classic_levels += ["vgaspec"+repr(i)+".dat" for i in range(0,4)]
    classic_styles  = ["ground"+repr(i)+"o.dat" for i in range(0,5)]
    classic_styles += ["main.dat"]
    if not (False in map(check, classic_levels+classic_styles)):
        return "Classic", ("Decrunch levels", "Upsample levels to Amiga quality", "Decrunch styles", "Use gzip compression")
    
    onml_levels  = ["dlvel00"+repr(i)+".dat" for i in range(0,10)]
    onml_levels += ["dlvel0"+repr(i)+".dat" for i in range(10,12)]
    onml_styles  = ["ground"+repr(i)+"o.dat" for i in range(0,4)]
    onml_styles += ["main.dat"]
    if not (False in map(check, onml_levels+onml_styles)):
        return "Oh No", ("Decrunch levels", "Decrunch styles", "Use gzip compression")
    
    return g.msg("decrunch_nogame"), None
    
convertPalette = lambda p: "".join(["".join(map(chr,m)) for m in p])

def getEGAPrevVal(byte):
    test = ord(byte)
    largeEnd = test%8
    smallEnd = test/8
    return (64*(smallEnd/4) + 191*(largeEnd/4),
            64*((smallEnd%4)/2) + 191*((largeEnd%4)/2),
            64*(smallEnd%2) + 191*(largeEnd%2))
    
getVGAColour = lambda (r,g,b): (int(round(r*255/63.)), int(round(g*255/63.)), int(round(b*255/63.)))

def deplanarize(source, startPos, width, height, bpp, rawOut=False, numberOfFrames=1, palette=None, mask=False):
    state = {"source":source,"index":startPos,"bitIndex":0,
             "currBits":reverseBits(ord(source[startPos]))}
    frameSize = width*height*bpp/8 
    if mask:  # this is required for interacts, as the planar graphic is a bit bigger
        frameSize = frameSize*5/4
        
    rawImage = [0]*width*height*numberOfFrames
    for h in range(0,numberOfFrames): # iterator used for animation
        state = {"source":source,"index":startPos+h*frameSize,"bitIndex":0,
             "currBits":reverseBits(ord(source[startPos+h*frameSize]))}
        for i in range(0,bpp):
            for j in range(0,width*height):
                rawImage[h*width*height+j] += getBit(state) << i
            
    if rawOut: # release an array of raw palette values
        return rawImage
    else: # release a python image object (recommended)
        niceImage = Image.new("P",(width,height*numberOfFrames),0)
        niceImage.putdata(rawImage)
        if palette!=None: # use a palette, save some time
            niceImage.putpalette(convertPalette(palette));
        return niceImage

def ripTitle(title, path, args):
    if title == "Classic":
        g.LEMMINGS_PATH = os.path.join(path,"")
        if args[0]: # Rip levels
            ripClassic(patched=args[1],gzip=args[3]) # upsample to amiga
        if args[2]:
            fullAuto() # rip styles
    elif title == "Oh No":
        g.ONML_PATH = os.path.join(path,"")
        if args[0]: # Rip levels
            ripONML(gzip=args[2])
        if args[1]:
            fullAutoONML() # rip styles
    print "All done! We now return back to the GUI..."
    return # jorb well done

def fullAuto():
    for i in range(0,5):
        print "*** Decrunching graphics set", i
        e = exportDOSStyle(getFile("ground"+repr(i)+"o.dat", g.LEMMINGS_PATH), getFile("vgagr"+repr(i)+".dat", g.LEMMINGS_PATH), i)
    print "*** Decrunching main data"
    extractDOSMain(getFile("main.dat", g.LEMMINGS_PATH))
    return

def fullAutoONML():
    for i in range(0,4):
        print "*** Decrunching graphics set", 5+i
        e = exportDOSStyle(getFile("ground"+repr(i)+"o.dat", g.ONML_PATH), getFile("vgagr"+repr(i)+".dat", g.ONML_PATH), i, ONML=True)
    print "*** Decrunching main data"
    extractDOSMain(getFile("main.dat", g.ONML_PATH))
    return

def extractDOSMain(main):
    """extractDOSMain(main_string) -> Extracts the important bits from a main.dat file.
    """
    main = decompressDOSDATString(main)
    # section 0: lemming anims
    prefix = g.RESOURCES_PATH+"pixmaps/"
    
    # lemming animations from main[0]: format is [offset,height,width,bpp,numberOfFrames,name]
    lemming = [ [0x0000,16,10,2,8,"s_walker_r"],
    [0x0140,16,10,2,1,"s_bounder_r"],
    [0x0168,16,10,2,8,"s_walker_l"],
    [0x02A8,16,10,2,1,"s_bounder_l"],
    [0x02D0,16,14,3,16,"s_digger"],
    [0x0810,16,12,2,8,"s_actclimber_r"],
    [0x0990,16,12,2,8,"s_actclimber_l"],
    [0x0B10,16,10,2,16,"s_drowner"],
    [0x0D90,16,12,2,8,"s_postclimber_r"],
    [0x0F10,16,12,2,8,"s_postclimber_l"],
    [0x1090,16,13,3,16,"s_builder_r"],
    [0x1570,16,13,3,16,"s_builder_l"],
    [0x1A50,16,10,3,32,"s_basher_r"],
    [0x21D0,16,10,3,32,"s_basher_l"],
    [0x2950,16,13,3,24,"s_miner_r"],
    [0x30A0,16,13,3,24,"s_miner_l"],
    [0x37F0,16,10,2,4,"s_faller_r"],
    [0x3890,16,10,2,4,"s_faller_l"],
    [0x3930,16,16,3,4,"s_prefloater_r"],
    [0x3AB0,16,16,3,4,"s_actfloater_r"],
    [0x3C30,16,16,3,4,"s_prefloater_l"],
    [0x3DB0,16,16,3,4,"s_actfloater_l"],
    [0x3F30,16,10,2,16,"s_splatter"],
    [0x41B0,16,13,2,8,"s_leaver"],
    [0x4350,16,14,4,14,"s_burner"],
    [0x4970,16,10,2,16,"s_blocker"],
    [0x4BF0,16,10,2,8,"s_shrugger_r"],
    [0x4D30,16,10,2,8,"s_shrugger_l"],
    [0x4E70,16,10,2,16,"s_goner"],
    [0x50F0,32,32,3,1,"s_exploder"] ]
    for l in lemming:
        deplanarize(main[0],l[0],l[1],l[2],l[3],numberOfFrames=l[4],palette=default_pal).save(prefix+"lemmings/"+l[5]+"-h"+repr(l[2])+".png" , optimize=1, transparency=0)

    masks = [ [0x0000,16,10,1,4,"s_basher_mr"],
    [0x0050,16,10,1,4,"s_basher_ml"],
    [0x00A0,16,13,1,2,"s_miner_mr"],
    [0x00D4,16,13,1,2,"s_miner_ml"],
    [0x0108,16,22,1,1,"s_exploder_m"] ]
    for m in masks:
        deplanarize(main[1],m[0],m[1],m[2],m[3],numberOfFrames=m[4],palette=masks_pal).save(prefix+"lemmings/"+m[5]+"-h"+repr(m[2])+".png" , optimize=1, transparency=0)

    numbers = [ [0x0134,8,8,1,1,"bomb_9"],
    [0x013C,8,8,1,1,"bomb_8"],
    [0x0144,8,8,1,1,"bomb_7"],
    [0x014C,8,8,1,1,"bomb_6"],
    [0x0154,8,8,1,1,"bomb_5"],
    [0x015C,8,8,1,1,"bomb_4"],
    [0x0164,8,8,1,1,"bomb_3"],
    [0x016C,8,8,1,1,"bomb_2"],
    [0x0174,8,8,1,1,"bomb_1"],
    [0x017C,8,8,1,1,"bomb_0"] ]
    for n in numbers:
        deplanarize(main[1],n[0],n[1],n[2],n[3],numberOfFrames=n[4],palette=numbers_pal).crop((0,3,5,8)).save(prefix+"count/"+n[5]+".png" , optimize=1, transparency=0)

    #deplanarize(main[4],0x2A00,48,16,4,numberOfFrames=16,palette=menu_pal).save(prefix+"s_cranker_l.png")
    #deplanarize(main[4],0x4200,48,16,4,numberOfFrames=16,palette=menu_pal).save(prefix+"s_cranker_r.png")
    
    return    

def extractDOSSpecial(vgaspec, pal="vga"):
    """extractDOSSpecial(vgaspec_string) -> Extracts the images from the special
    levels of DOS Lemmings.
    
    Code mostly guessed by running ccexplore's myvgaspec utility several times. 
    The things I do to avoid writing a DAT recompressor...

    vgaspec0: What an AWESOME level
    vgaspec1: MENACING!
    vgaspec2: A Beast2 of a Level
    vgaspec3: A Beast of a Level
"""
    vgaspec = decompressDOSDATString(vgaspec)[0]

    palette = {"vga":[],"vga_prev":[],"ega":[],"ega_prev":[]}
    for i in range(0x00,0x18,0x03):
        palette["vga"].append(getVGAColour(
                   (ord(vgaspec[i]),ord(vgaspec[i+1]),ord(vgaspec[i+2]))) )
        # VGA palette repeated here for good measure
        palette["vga_prev"].append(getVGAColour(
                   (ord(vgaspec[i]),ord(vgaspec[i+1]),ord(vgaspec[i+2]))) )
    for i in range(0x18,0x20):
        palette["ega"].append(getEGAPrevVal(vgaspec[i]))
    # this next palette appears unused, the EGA one seems to be used for both 
    # the level and preview representations. ah, what the hey, in it goes
    for i in range(0x20,0x28): 
        palette["ega_prev"].append(getEGAPrevVal(vgaspec[i]))
    
        
    counter = 0x28  # start of data section
    planar = []
    for i in range(0,4):
        planar.append("")
        while True:
            if ord(vgaspec[counter]) < 0x80:
                for j in range(0,ord(vgaspec[counter])+1):
                    counter += 1
                    planar[-1] += vgaspec[counter]
                counter += 1
            elif ord(vgaspec[counter]) == 0x80:
                counter += 1
                break
            else: # ord(vgaspec[counter]) > 0x80
                planar[-1] += vgaspec[counter+1]*(257-ord(vgaspec[counter]))
                counter += 2

    rawImage = []
    for p in planar:
        rawImage += deplanarize(p,0,960,40,3,rawOut=True)

    niceImage = Image.new("P",(960,160),0)
    niceImage.putdata(rawImage)
    niceImage.putpalette(convertPalette(palette[pal]))
    return niceImage
    
    

def exportDOSStyle(ground, vgagr, styleID, pal="vga", ONML=False):
    """exportDOSStyle(ground_string, vgagr_string) -> Extracts
    all the graphics contained in a ground/vgagr pair and converts it
    to Tundra's combined XML and PNG output.
    
    Big shoutouts to Lemmingologist, who provided the format details of 
    groundXo.dat and vgagrX.dat"""
    interacts = []
    terrains = []
    palette = {}
    vgagr = decompressDOSDATString(vgagr)
    if not ONML:
        sc = styles
    else:
        sc = styles_onml
    
    try:
        os.makedirs(g.RESOURCES_PATH+"styles/"+sc[styleID]+"/inter")
    except OSError:
        pass

    for i in range(0x000,0x1bf,28):
        if ground[i:i+28] != '\x00'*28:
            interacts.append({})
            interacts[-1]["objType"] = "unsure"
            if (ord(ground[i])==3) and (ord(ground[i+2])==1):
                interacts[-1]["objType"] = "entrance"
        
            interacts[-1]["numFrames"] = ord(ground[i+3])
            interacts[-1]["width"] = ord(ground[i+4])
            interacts[-1]["height"] = ord(ground[i+5])
            interacts[-1]["frameAndMaskSize"] = ord(ground[i+7])*0x100 + ord(ground[i+6])
            interacts[-1]["frameSize"] = ord(ground[i+9])*0x100 + ord(ground[i+8])
        
            if map(ord,ground[i+14:i+20])==[0,0,0,0,1,1]:
                interacts[-1]["activeRect"] = [0,0,0,0]
            elif map(ord,ground[i+14:i+20])==[1,0,0,0,0,1]:
                interacts[-1]["activeRect"] = [0,0,0,0]
                if not interacts[-1]["objType"]=="entrance":
                    print "Warning: bytes 14-19 claim object is an entrance,"
                    print "but type is different."
            else:
                # TODO: remember that the two high coordinates are, in fact, for the bottom left
                interacts[-1]["activeRect"] = [4*(ord(ground[i+15])*0x100+ord(ground[i+14])),
                                             4*(ord(ground[i+17])*0x100+ord(ground[i+16])),
                                             4*ord(ground[i+18]), 4*ord(ground[i+19])]
            

            if (ord(ground[i+20]) == 0) & (interacts[-1]["objType"] != "entrance"):
                interacts[-1]["objType"] = "harmless"
            elif ord(ground[i+20]) == 1:
                interacts[-1]["objType"] = "exit"
            elif ord(ground[i+20]) == 4:
                interacts[-1]["objType"] = "trap"
            elif ord(ground[i+20]) == 5:
                interacts[-1]["objType"] = "water"
            elif ord(ground[i+20]) == 6:
                interacts[-1]["objType"] = "hazard"
            elif ord(ground[i+20]) == 7:
                interacts[-1]["objType"] = "leftArrows"
            elif ord(ground[i+20]) == 8:
                interacts[-1]["objType"] = "rightArrows"
            else:
                interacts[-1]["objTypeNum"] = ground[i+20]            
    
            interacts[-1]["startPos"] = ord(ground[i+24])*0x100 + ord(ground[i+23])
            interacts[-1]["filePath"] = g.RESOURCES_PATH+"styles/"+sc[styleID]+"/inter/"+repr(len(interacts)-1)+".png"

    # HACK: workaround for really dodgy collision masks
    if sc[styleID] == "Marble":
        interacts[8]["activeRect"][1] -= 2            
    elif sc[styleID] == "Bubble":
        for i in [0,8,9]:
            interacts[i]["activeRect"][1] -= 4
            interacts[i]["activeRect"][3] += 4


    try:
        os.makedirs(g.RESOURCES_PATH+"styles/"+sc[styleID]+"/terra")
    except OSError:
        pass
        
    for i in range(0x1c0,0x3c0,8):
        if ground[i:i+8] != '\x00'*8:
            terrains.append({})
            terrains[-1]["width"] = ord(ground[i])
            terrains[-1]["height"] = ord(ground[i+1])
            terrains[-1]["startPos"] = ord(ground[i+3])*0x100 + ord(ground[i+2])
            terrains[-1]["frameSize"]= terrains[-1]["width"]*terrains[-1]["height"]/2
            terrains[-1]["filePath"] = g.RESOURCES_PATH+"styles/"+sc[styleID]+"/terra/"+repr(len(terrains)-1)+".png"
    
    palette["ega"] = [ega[0x00],ega[0x11],ega[0x02],ega[0x17],
                      ega[0x16],ega[0x14],ega[0x07],ega[0x14]]
    for i in map(ord,ground[0x3c0:0x3c8]):
        palette["ega"].append(ega[i])
    
    palette["ega_prev"] = []
    for i in range(0x3c8,0x3d8):
        palette["ega_prev"].append(getEGAPrevVal(ground[i]))
        
    palette["vga"] = map(getVGAColour,[(0,0,0),(16,16,56),(0,44,0),(60,52,52),
                                       (60,60,0),(60,8,8),(32,32,32),(48,16,24)])
    for i in range(0x3d8,0x3f0,3):
        palette["vga"].append(getVGAColour((ord(ground[i]), ord(ground[i+1]), ord(ground[i+2]))))
    # HACK: for some reason, that godawful pink colour in the main palette is never used
    # instead, it's assumed to be whatever colour the tileset's dirt colour is set as
    palette["vga"][7] = palette["vga"][8]

    palette["vga_prev"] = []
    for i in range(0x3f0,0x420,3):
        palette["vga_prev"].append(getVGAColour((ord(ground[i]), ord(ground[i+1]), ord(ground[i+2]))))
    
    for o in interacts:
        animation = [] # note: these anims have an extra mask bitplane, so the mask attribute must be set to True 
        if o["objType"]=="entrance":  # hack to fix the way trapdoor sprites are stored
            animation += deplanarize(vgagr[1],o["startPos"],
                                      o["width"],o["height"],4,rawOut=True,numberOfFrames=o["numFrames"]-1,mask=True)
            animation += deplanarize(vgagr[1],o["startPos"]-o["frameAndMaskSize"],
                                      o["width"],o["height"],4,rawOut=True,mask=True)
        else:
            animation += deplanarize(vgagr[1],o["startPos"],
                                      o["width"],o["height"],4,rawOut=True,numberOfFrames=o["numFrames"],mask=True)
        o["niceImage"] = Image.new("P",(o["width"],o["height"]*o["numFrames"]),0)
        o["niceImage"].putdata(animation)
        o["niceImage"].putpalette(convertPalette(palette[pal]))
        o["niceImage"].save(o["filePath"], optimize=1, transparency=0)
    
    for t in terrains:
        t["niceImage"] = deplanarize(vgagr[0],t["startPos"],t["width"],t["height"],4,palette=palette[pal])
        t["niceImage"].save(t["filePath"], optimize=1, transparency=0)
    # XML dumping stage
    f = open(g.RESOURCES_PATH+"styles/"+sc[styleID]+".xml","w")

    xmlinfo = xml.sax.writer.XMLDoctypeInfo()
    xmlinfo.add_element_container("style")
    xmlinfo.add_element_container("interDefs")
    xmlinfo.add_element_container("terraDefs")
    xmlout = xml.sax.writer.PrettyPrinter( f, dtdinfo=xmlinfo ) 
    xmlout.startDocument()
    xmlout.startElement("style", {"styleName":sc[styleID], "author":"DMA Design", "version":"1.0", "scale":"1", "favColour": rgbToPrintable(palette["vga"][7]) })
    xmlout.startElement("interDefs",{})
    for i in interacts:
        xmlout.startElement("interDef", {"path":i["filePath"][len(g.RESOURCES_PATH+"styles/"):] , "frameHeight": i["height"], "interType":i["objType"], "rectX":repr(i["activeRect"][0]), "rectY":repr(i["activeRect"][1]), "rectW":repr(i["activeRect"][2]), "rectH":repr(i["activeRect"][3])  })
        xmlout.endElement("interDef")
    xmlout.endElement("interDefs")
    
    xmlout.startElement("terraDefs",{})
    for t in terrains:        
        xmlout.startElement("terraDef",{"path":t["filePath"][len(g.RESOURCES_PATH+"styles/"):]})
        xmlout.endElement("terraDef")
    xmlout.endElement("terraDefs")
    xmlout.endElement("style")
    xmlout.endDocument()
    f.close()
    return # {"interact":interacts, "terrain":terrains, "palette":palette}

def decompressDOSDATString(source):
    """decompressDOSDATSTRING(string) -> Decompresses a DAT string into
    a list of several smaller, uncompressed strings.
    
    This method was pythonated from ccexplore's "ldecomp.c". ccexplore
    is an absolutely corking chap, please don't hesitate to shower him 
    with beer and kisses."""
    totalNumBytes = len(source)
    
    def getNextBits(n,state):
        result = 0
        while True:
            state["bitcnt"] -= 1
            if state["bitcnt"] == 0:
                state["cptr"] -= 1
                state["curbits"] = ord(state["cdata"][state["cptr"]])
                state["checksum"] ^= state["curbits"]
                state["bitcnt"] = 8
            result <<= 1;
            result |= (state["curbits"] & 1)
            state["curbits"] >>= 1;
            
            n -= 1
            if not n>0: 
                break
        return result
    
    def copyPrevData(blocklen,offset_size,state):
        offset = getNextBits(offset_size, state)
        for i in range(0,blocklen):
            state["dptr"] -= 1
            state["ddata"][state["dptr"]] = state["ddata"][state["dptr"]+offset+1]
        return
    
    def dumpData(numbytes,state):
        while True:
            state["dptr"] -= 1
            state["ddata"][state["dptr"]] = getNextBits(8,state)
            
            numbytes -= 1
            if not numbytes>0:
                break
        return
    
    arrToString = lambda array: "".join(map(chr,array))  # ONE LINE! ONE SODDING LINE!

    
    pointer = 0;
    target = []
    keepLooping = True
    while keepLooping:
        bitCnt = ord(source[pointer])
        checksum = ord(source[pointer+1])
        decompressedSize = ord(source[pointer+4])*0x100 + ord(source[pointer+5])
        compressedSize = ord(source[pointer+8])*0x100 + ord(source[pointer+9])
        
        print "----   HEADER   ----"
        print "bitCnt = "+repr(bitCnt)
        print "checksum = "+repr(checksum)
        print "decompressedSize = "+repr(decompressedSize)
        print "compressedSize = "+repr(compressedSize)
        print "---- END HEADER ----"
    
        print "Decompressing data...",
        
        pointer += 10
        totalNumBytes -= 10
        compressedSize -= 10
        
        compressedData = source[pointer:pointer+compressedSize]
        pointer += compressedSize
        
        totalNumBytes -= compressedSize
    
        state = {"bitcnt":bitCnt+1, "cptr":(compressedSize-1), "dptr":decompressedSize,
                 "cdata": compressedData, "ddata": [0]*decompressedSize, # <- OH HEAVENS NO
                 "curbits": ord(compressedData[-1]), "checksum": ord(compressedData[-1])}
        
        while True:
            if getNextBits(1,state)==1:
                test = getNextBits(2,state)
                if test==0:
                    copyPrevData(3,9,state)
                elif test==1:
                    copyPrevData(4,10,state)
                elif test==2:
                    copyPrevData(getNextBits(8,state)+1,12,state)
                elif test==3:
                    dumpData(getNextBits(8,state)+9,state)
            else:
                test = getNextBits(1,state)
                if test==0:
                    dumpData(getNextBits(3,state)+1,state)
                elif test==1:
                    copyPrevData(2,8,state)
            
            if not (state["dptr"]>0): 
                break
        #arse
        print "done!"
                
        if checksum != state["checksum"]:
            print "Warning: checksum doesn't match header"
        
        target.append(arrToString(state["ddata"]))  
        if not totalNumBytes > 0:
            break
        
    return target
    

def pythonateDOSLevelString(source, author="DMA Design", ONML=False):
    """pythonateDOSLevelString(string) -> Takes a string containing a
    raw level file and spits out a Level object."""
    target = Level()
    # NOTE: next bits annotated in hex, for those who wish to
    # play with raw level files
    target.releaseRate = ord(source[0x0001])
    target.numReleased = ord(source[0x0003])
    target.numToSave = ord(source[0x0005])
    target.timeLimit = ord(source[0x0007])*60
    target.numSkills["s_climber"] = ord(source[0x0009])
    target.numSkills["s_floater"] = ord(source[0x000b])
    target.numSkills["s_bomber"] = ord(source[0x000d])
    target.numSkills["s_blocker"] = ord(source[0x000f])
    target.numSkills["s_builder"] = ord(source[0x0011])
    target.numSkills["s_basher"] = ord(source[0x0013])
    target.numSkills["s_miner"] = ord(source[0x0015])
    target.numSkills["s_digger"] = ord(source[0x0017])
    target.startPosX = ord(source[0x0018])*0x100 + ord(source[0x0019])
    if not ONML:        
        target.defaultStyle = styles[ord(source[0x001b])]
    else:
        target.defaultStyle = styles_onml[ord(source[0x001b])]    

    for i in range(0x0020,0x011f,8):
        if source[i:i+8] != ("\x00"*8):
            inter = InteractObj()
            
            temp = ord(source[i])*0x100 + ord(source[i+1])
            if temp > 0xf000:
                temp -= 0x10000
            temp -= 0x10;
            inter.posX = temp
            
            temp = ord(source[i+2])*0x100 + ord(source[i+3])
            if temp > 0xf000:
                temp -= 0x10000 
            inter.posY = temp
            
            # TODO: figure out object index
            inter.itemID = ord(source[i+5])
            inter.style = target.defaultStyle
            
            if ord(source[i+6])==0x40 or ord(source[i+6])==0xc0:
                inter.drawFullGraphic = False
                inter.drawIfTerrainUnderneath = True
            elif ord(source[i+6])==0x80:
                inter.drawFullGraphic = False
                inter.drawButDontOverwrite = True
            

            if ord(source[i+7])==0x8F:
                inter.drawUpsideDown = True
            
            target.interacts.append(inter)
    
    for i in range(0x0120,0x075f,4):
        if source[i:i+4] != ("\xff"*4):
            terra = TerrainObj()
            terra.posX = (ord(source[i])*0x100 + ord(source[i+1]))%0x1000 - 0x10
            
            temp = (ord(source[i+2])*0x10 + (ord(source[i+3])/0x80)*0x8 )
            if (temp > 0xe00):
                temp -= 0x1000
            terra.posY = (temp-0x20)/0x8
            # TODO: nut out object index
            terra.itemID = ord(source[i+3])%0x40
            terra.style = target.defaultStyle
            target.terrains.append(terra)
            
            flags = ord(source[i])/0x10
            if not(flags == 0):
                terra.drawFullGraphic = False
                terra.drawButDontOverwrite = bool(flags/0x8)
                terra.drawUpsideDown = bool((flags%0x8)/0x4)
                terra.drawSubtractiveTerrain = bool((flags%0x4)/0x2)
                if terra.drawButDontOverwrite:
                    terra.drawSubtractiveTerrain = False            
            
    for i in range(0x0760,0x7df,4):
        if source[i:i+4] != ("\x00"*4):
            steel = Rect(0,0,16,16)
            steel.left = (((((ord(source[i])<<8) + ord(source[i+1]))>>7) - 4)<<2)
            steel.top = ((((ord(source[i])<<8) + ord(source[i+1])) % 0x80)<<2)
            steel.width = (((ord(source[i+2])>>4)+1)<<2)
            steel.height = ((ord(source[i+2])%0x10 + 1)<<2)
            target.steels.append(steel)

    target.levelName = source[0x7e0:0x800].strip()
    target.author = author
    # TODO: BYTES 0x001C to 0x001D Extended Graphic Set
    # probably : vgaspec(source[0x1d]-1)
    # source[0x01d] == 0 when not special
    if not ord(source[0x01d]) == 0:     
        target.exportSpecialFlag = ord(source[0x01d])

    return target





def exportSpecial(target, filename):
    folder = os.path.split(filename)[0] + os.path.sep + os.path.split(filename)[1].split(".")[0]
    try:
        os.makedirs(folder+"/inter")
        os.makedirs(folder+"/terra")
    except OSError:
        pass

    # if my software engineering lecturer ever reads this, can I just begin by claiming that
    # lack of interest prevailed over the idea of writing everything again to be recyclable,
    # which is why so much of this is cloned from extractDOSStyle rather than subdivided into
    # autonomous get and set functions written in triplicate. Oh, and unfortunately no UML
    # diagram is available. sorry about that.
    #
    # if it's any consolation, I did eventually get bitten on the arse by this scrimshaw code

    scene = extractDOSSpecial(getFile("vgaspec"+repr(target.exportSpecialFlag-1)+".dat", g.LEMMINGS_PATH))
    scene.save(folder+"/terra/0.png", optimize=1, transparency=0)    

    interacts = []          # cloned code used here to extract the Soil animations! oh NO! 
    #CHANGE FROM STOCK:
	# ground = getFile("ground0o.dat", g.LEMMINGS_PATH)
    if target.defaultStyle in styles_onml:
       ground = getFile("ground"+repr(styles_onml.index(target.defaultStyle))+"o.dat", g.LEMMINGS_PATH)
    elif target.defaultStyle in styles:
        ground = getFile("ground"+repr(styles.index(target.defaultStyle))+"o.dat", g.LEMMINGS_PATH)
    else:
        ground = getFile("ground0o.dat", g.LEMMINGS_PATH)
	#END CHANGE FROM STOCK

    bg = TerrainObj()
    bg.posX = 304
    bg.posY = 0
    bg.itemID = 0
    bg.style = "self"
    target.terrains.append(bg)

    target.keyword = "special"+repr(target.exportSpecialFlag-1) # needed for music.py

    for i in range(0x000,0x1bf,28):
        if ground[i:i+28] != '\x00'*28:
            interacts.append({})
            interacts[-1]["objType"] = "unsure"
            if (ord(ground[i])==3) and (ord(ground[i+2])==1):
                interacts[-1]["objType"] = "entrance"
        
            interacts[-1]["numFrames"] = ord(ground[i+3])
            interacts[-1]["width"] = ord(ground[i+4])
            interacts[-1]["height"] = ord(ground[i+5])
            interacts[-1]["frameAndMaskSize"] = ord(ground[i+7])*0x100 + ord(ground[i+6])
            interacts[-1]["frameSize"] = ord(ground[i+9])*0x100 + ord(ground[i+8])
        
            if map(ord,ground[i+14:i+20])==[0,0,0,0,1,1]:
                interacts[-1]["activeRect"] = [0,0,0,0]
            elif map(ord,ground[i+14:i+20])==[1,0,0,0,0,1]:
                interacts[-1]["activeRect"] = [0,0,0,0]
                if not interacts[-1]["objType"]=="entrance":
                    print "Warning: bytes 14-19 claim object is an entrance,"
                    print "but type is different."
            else:
                # TODO: remember that the two high coordinates are, in fact, for the bottom left
                interacts[-1]["activeRect"] = [4*(ord(ground[i+15])*0x100+ord(ground[i+14])),
                                             4*(ord(ground[i+17])*0x100+ord(ground[i+16])),
                                             4*ord(ground[i+18]), 4*ord(ground[i+19])]
            
            if (ord(ground[i+20]) == 0) & (interacts[-1]["objType"] != "entrance"):
                interacts[-1]["objType"] = "harmless"
            elif ord(ground[i+20]) == 1:
                interacts[-1]["objType"] = "exit"
            elif ord(ground[i+20]) == 4:
                interacts[-1]["objType"] = "trap"
            elif ord(ground[i+20]) == 5:
                interacts[-1]["objType"] = "water"
            elif ord(ground[i+20]) == 6:
                interacts[-1]["objType"] = "hazard"
            elif ord(ground[i+20]) == 7:
                interacts[-1]["objType"] = "leftArrows"
            elif ord(ground[i+20]) == 8:
                interacts[-1]["objType"] = "rightArrows"
            else:
                interacts[-1]["objTypeNum"] = ground[i+20]            
    
            interacts[-1]["startPos"] = ord(ground[i+24])*0x100 + ord(ground[i+23])
            interacts[-1]["filePath"] = folder + "/inter/"+repr(len(interacts)-1)+".png"
	#CHANGE FROM STOCK
    #v = decompressDOSDATString(getFile("vgagr0.dat", g.LEMMINGS_PATH))[1]
    if target.defaultStyle in styles_onml:
        v = decompressDOSDATString(getFile("vgagr"+repr(styles_onml.index(target.defaultStyle))+".dat", g.LEMMINGS_PATH))[1]
    elif target.defaultStyle in styles:
        v = decompressDOSDATString(getFile("vgagr"+repr( styles.index(target.defaultStyle))+".dat", g.LEMMINGS_PATH))[1]
    else:
        v = decompressDOSDATString(getFile("vgagr0.dat", g.LEMMINGS_PATH))[1]
	#END CHANGE FROM STOCK
    for o in interacts:
        animation = [] # note: these anims have an extra mask bitplane, so the mask attribute must be set to True 
        if o["objType"]=="entrance":  # hack to fix the way trapdoor sprites are stored
            animation += deplanarize(v,o["startPos"],
                                      o["width"],o["height"],4,rawOut=True,numberOfFrames=o["numFrames"]-1,mask=True)
            animation += deplanarize(v,o["startPos"]-o["frameAndMaskSize"],
                                      o["width"],o["height"],4,rawOut=True,mask=True)
        else:
            animation += deplanarize(v,o["startPos"],
                                      o["width"],o["height"],4,rawOut=True,numberOfFrames=o["numFrames"],mask=True)
        o["niceImage"] = Image.new("P",(o["width"],o["height"]*o["numFrames"]),0)
        o["niceImage"].putdata(animation)
        x = [] 
        for d in default_pal:
            x += d
        o["niceImage"].putpalette(x + scene.getpalette()[:24])
        o["niceImage"].save(o["filePath"], optimize=1, transparency=0)
    
    f = open(folder+"/style.xml","w")

    xmlinfo = xml.sax.writer.XMLDoctypeInfo()
    xmlinfo.add_element_container("style")
    xmlinfo.add_element_container("interDefs")
    xmlinfo.add_element_container("terraDefs")
    xmlout = xml.sax.writer.PrettyPrinter( f, dtdinfo=xmlinfo ) 
    xmlout.startDocument()
    xmlout.startElement("style", {"styleName":"Custom Level "+repr(target.exportSpecialFlag), "author":"DMA Design", "version":"1.0", "scale":"1"})
    xmlout.startElement("interDefs",{})
    for i in interacts:
        xmlout.startElement("interDef", {"path":"inter/"+os.path.split(i["filePath"])[1] , "frameHeight": i["height"], "interType":i["objType"], "rectX":repr(i["activeRect"][0]), "rectY":repr(i["activeRect"][1]), "rectW":repr(i["activeRect"][2]), "rectH":repr(i["activeRect"][3])  })
        xmlout.endElement("interDef")
    xmlout.endElement("interDefs")
    
    xmlout.startElement("terraDefs",{}) # only one!
    xmlout.startElement("terraDef",{"path": "terra/0.png" })
    xmlout.endElement("terraDef")

    xmlout.endElement("terraDefs")
    xmlout.endElement("style")
    xmlout.endDocument()
    f.close()

    target.defaultStyle = "self"
    for i in target.interacts:
        i.style = "self"
    return


def ripClassic(patched=True, gzip=False):
    levelSrc = [decompressDOSDATString(getFile("level00"+repr(i)+".dat", g.LEMMINGS_PATH)) for i in range(0,10)]
    oddtable = getFile("oddtable.dat", g.LEMMINGS_PATH)
    if patched:
        levelSrc, oddtable = patchClassic(levelSrc, oddtable)    
    levelList,levelListCopy = [],[]
    for s in levelSrc:
        levelList.append([pythonateDOSLevelString(x) for x in s])
        levelListCopy.append([pythonateDOSLevelString(x) for x in s])
    orderedLevels = sliceForClassic(levelList, levelListCopy, oddtable)
    
    for i in range(0,len(orderedLevels)):
        try:
            os.makedirs(g.RESOURCES_PATH+"levels/Classic/"+repr(i)+" "+difficulty[i])        
        except OSError:
            pass            
        for j in range(0,len(orderedLevels[i])):
            exportFileName = g.RESOURCES_PATH+"levels/Classic/"+repr(i)+" "+difficulty[i]+"/"+repr(j)+".xml"
            if gzip: exportFileName += ".gz"
            if hasattr(orderedLevels[i][j], "exportSpecialFlag"):
                exportSpecial(orderedLevels[i][j], exportFileName)
            orderedLevels[i][j].exportXML(exportFileName, gzip)

def ripONML(gzip=False):
    levelSrc = [decompressDOSDATString(getFile("dlvel00"+repr(i)+".dat",g.ONML_PATH)) for i in range(0,10)]
    levelSrc += [decompressDOSDATString(getFile("dlvel0"+repr(i)+".dat",g.ONML_PATH)) for i in range(10,13)]
    levelList = []    
    for s in levelSrc:
        levelList.append([pythonateDOSLevelString(x,ONML=True) for x in s])
    orderedLevels = sliceForONML(levelList)
    
    for i in range(0,len(orderedLevels)):
        try:
            os.makedirs(g.RESOURCES_PATH+"levels/Oh No/"+repr(i)+" "+difficulty_onml[i])        
        except OSError:
            pass            
        for j in range(0,len(orderedLevels[i])):
            exportFileName = g.RESOURCES_PATH+"levels/Oh No/"+repr(i)+" "+difficulty_onml[i]+"/"+repr(j)+".xml"
            if gzip: exportFileName += ".gz"
            orderedLevels[i][j].exportXML(exportFileName, gzip)


# note to self: make this optional
# as in, allow the person to choose either amiga graphics
# amiga skillnums, or both
def patchClassic(levelSrc, oddtable):
    import amigapatchtables as a
    patchedSrc = []
    for i in range(0,len(levelSrc)):
        patchedSrc.append([])
        for j in range(0,len(levelSrc[i])):
            patchedSrc[i].append(levelSrc[i][j])
            offset = 0
            for p in a.CLASSIC_PATCH_TABLE[i][j]:
                patchedSrc[i][j] = patchedSrc[i][j][:p[1]+offset] +p[3]+ patchedSrc[i][j][p[2]+offset:]
                if p[0] == "insert":
                    offset += len(p[3])
                elif p[0] == "delete":
                    offset -= p[2] - p[1]
                elif p[0] == "replace":
                    offset -= (p[2] - p[1])- len(p[3])
            # offset should == 0

    patchedTable = oddtable
    offset = 0
    for p in a.CLASSIC_ODDTABLE_PATCH_TABLE:
        patchedTable = patchedTable[:p[1]+offset] +p[3]+ patchedTable[p[2]+offset:]
        if p[0] == "insert":
            offset += len(p[3])
        elif p[0] == "delete":
            offset -= p[2] - p[1]
        elif p[0] == "replace":
            offset -= (p[2] - p[1])- len(p[3])
    # offset should == 0 by now
    return (patchedSrc, patchedTable)


# if there's some madcap way of extracting the level play order, it hasn't been helpfully written down
# therefore, I spent two days untangling which bloody order levels are shown in      
def sliceForClassic(levelList, levelListCopy, oddTable):
    sorted = [[],[],[],[]]
    sortmap = [[],[],[],[]]

    oddmap = []
    for i in range(0,4480,56):
        oddmap.append(oddTable[i:i+56])

    # a preceeding -1 denotes use of the oddTable
    sortmap[0] = [[9,1,5,6,2,3,4,7],[-1,6,0,6],[-1,10,1,2],[-1,26,3,2],[-1,34,4,2],[-1,7,0,7],[1,6],[-1,15,1,7],
                  [-1,18,2,2],[-1,20,2,4],[-1,23,2,7],[-1,35,4,3],[-1,41,5,1],[-1,51,6,3],[-1,68,8,4],[1,3],[-1,33,4,1],
                  [-1,47,5,7],[-1,48,6,0],[-1,57,7,1],[-1,38,4,6],[-1,49,6,1],[-1,53,6,5],[-1,66,8,2]]
    sortmap[1] = [[0,0],[-1,14,1,6],[-1,17,2,1],[-1,24,3,0],[-1,25,3,1],[-1,27,3,3],[-1,28,3,4],[-1,39,4,7],[-1,50,6,2],
                  [-1,59,7,3],[-1,63,7,7],[-1,64,8,0],[-1,67,8,3],[0,2],[-1,73,9,1],[-1,75,9,3],[-1,76,9,4],[-1,77,9,5],
                  [-1,79,9,7],[0,3,5,6,7],[1,0,1,2,4,5],[2,0],[1,7]]
    sortmap[2] = [[2,2,3,4,5,6,7],[3,0,1,2,3,4,5,6,7],[0,1],[4,0,1,2,3,4,5,6,7],[5,0,1,2,3,4],[2,1],[6,7]]
    sortmap[3] = [[5,5,6,7],[6,0,1,2,3,4,5,6],[-1,55,6,7],[7,0,1,2,3,4,5,6,7],[-1,74,9,2],[8,0],[0,4],[8,1,2,3,4,5,6,7],
                  [9,0]]

    for i in range(0,4):
        for s in sortmap[i]:
            if s[0] == -1:            
                sorted[i] += [levelListCopy[s[2]][s[3]]]
                sorted[i][-1].releaseRate = ord(oddmap[s[1]][1])
                sorted[i][-1].numReleased = ord(oddmap[s[1]][3])
                sorted[i][-1].numToSave = ord(oddmap[s[1]][5])
                sorted[i][-1].timeLimit = ord(oddmap[s[1]][7])*60
                sorted[i][-1].numSkills["s_climber"] = ord(oddmap[s[1]][9])
                sorted[i][-1].numSkills["s_floater"] = ord(oddmap[s[1]][11])
                sorted[i][-1].numSkills["s_bomber"] = ord(oddmap[s[1]][13])
                sorted[i][-1].numSkills["s_blocker"] = ord(oddmap[s[1]][15])
                sorted[i][-1].numSkills["s_builder"] = ord(oddmap[s[1]][17])
                sorted[i][-1].numSkills["s_basher"] = ord(oddmap[s[1]][19])
                sorted[i][-1].numSkills["s_miner"] = ord(oddmap[s[1]][21])
                sorted[i][-1].numSkills["s_digger"] = ord(oddmap[s[1]][23])
                sorted[i][-1].levelName = oddmap[s[1]][24:56].strip()
            else:            
                sorted[i] += [levelList[s[0]][x] for x in s[1:]]

    return sorted


def sliceForONML(levelList):
    sorted = [[],[],[],[],[]] # tame crazy wild wicked havoc
    sortmap = [[],[],[],[]] # crazy wild wicked havoc
    sorted[0] = levelList[10] + levelList[11] + levelList[12] # easy peasy
    
    sortmap[0] = [[0,1],[1,0,4],[2,0,1],[3,0,1,5],[5,1,4,7],[6,7],[0,4],[1,5,6],[2,6],[3,4,7],[4,3],[6,4]]
    sortmap[1] = [[9,2],[7,0,1,2,3,5],[8,6],[0,7],[2,2,5],[3,3,6],[4,0,2,4],[5,0],[6,3,5,6],[4,7]]
    sortmap[2] = [[9,6],[4,6],[9,0,1],[0,5],[9,4],[6,1],[0,6],[1,2],[8,7],[4,1],[5,5],[6,0,2],[7,7],[8,1],[9,7,3],
                  [8,0],[7,6]]
    sortmap[3] = [[7,4],[5,3],[3,2],[2,7,4,3],[5,2],[1,7,1],[0,3,2,0],[4,5],[8,5],[1,3],[8,2,3],[5,6],[8,4],[9,5]]

    for i in range(0,4):
        for s in sortmap[i]:
            sorted[i+1] += [levelList[s[0]][x] for x in s[1:]]
    
    #HACK: changing a really long levelname
    sorted[4][8].levelName = '"Bugger."'
    return sorted 
