ForgePatch/fml/install/fml.py

import os, os.path, sys
import urllib, zipfile
import shutil, glob, fnmatch
import subprocess, logging, re, shlex
import csv
from hashlib import md5  # pylint: disable-msg=E0611

def download_deps(mcp_path):
    bin_path = os.path.normpath(os.path.join(mcp_path, 'runtime', 'bin'))
    ff_path = os.path.normpath(os.path.join(bin_path, 'fernflower.jar'))
    ret = True
    
    if not os.path.isfile(ff_path):
        try:
            urllib.urlretrieve("http://goo.gl/PnJHp", 'fernflower.zip')
            zf = zipfile.ZipFile('fernflower.zip')
            zf.extract('fernflower.jar', bin_path)
            zf.close()
            os.remove('fernflower.zip')
            print "Downloaded Fernflower successfully"
        except:
            print "Downloading Fernflower failed download manually from http://goo.gl/PnJHp"
            ret = False
    
    for lib in ['argo-2.25.jar', 'guava-12.0.1.jar', 'guava-12.0.1-sources.jar', 'asm-all-4.0.jar', 'asm-all-4.0-source.jar']:
        libF = os.path.join(mcp_path, 'lib')
        if not os.path.isdir(libF):
            os.makedirs(libF)
            
        target = os.path.normpath(os.path.join(libF, lib))
        
        if not os.path.isfile(target):
            try:
                urllib.urlretrieve('http://files.minecraftforge.net/fmllibs/' + lib, target)
                print 'Downloaded %s successfully' % lib
            except:
                print 'Download %s failed, download manually from http://files.minecraftforge.net/fmllibs/%s or http://files.minecraftforge.net/fmllibs/fml_libs_dev.zip and place in MCP/lib' % (lib, lib)
                ret = False
    
    return ret
        
def pre_decompile(mcp_dir, fml_dir):
    file_backup(os.path.join(mcp_dir, 'jars', 'bin'), 'minecraft.jar')
    file_backup(os.path.join(mcp_dir, 'jars'), 'minecraft_server.jar')
    
    client_jar = os.path.join(mcp_dir, 'jars', 'bin', 'minecraft.jar.backup')
    server_jar = os.path.join(mcp_dir, 'jars', 'minecraft_server.jar.backup')
    
    if not os.path.isfile(client_jar):
        print 'Could not find Client jar, decompile requires both client and server.'
        sys.exit(1)
    
    if not os.path.isfile(server_jar):
        print 'Could not find Server jar, decompile requires both client and server.'
        #sys.exit(1)
        
    md5_c = ""
    md5_s = ""
    with open(client_jar, 'rb') as fh:
        md5_c = md5(fh.read()).hexdigest()
    with open(server_jar, 'rb') as fh:
        md5_s = md5(fh.read()).hexdigest()
        
    clean_c = "771175c01778ea67395bc6919a5a9dc5"
    clean_s = "a726380b5546635662d8706dae0586c7"
    
    if not md5_c == clean_c:
        print 'Warning, Modified Client jar detected'
        print 'Continuing with decompile may produce unpredictable results'
        print 'If you continue, do not come to the FML or Forge team with decompile issues.'
        answer = raw_input('If you really want to continue, enter "Yes" ')
        if answer.lower() not in ['yes', 'y']:
            print 'You have not entered "Yes", aborting the decompile process'
            sys.exit(1)
            
    if not md5_s == clean_s:
        print 'Warning, Modified Server jar detected'
        print 'Continuing with decompile may produce unpredictable results'
        print 'If you continue, do not come to the FML or Forge team with decompile issues.'
        answer = raw_input('If you really want to continue, enter "Yes" ')
        if answer.lower() not in ['yes', 'y']:
            print 'You have not entered "Yes", aborting the decompile process'
            sys.exit(1)
    
def file_backup(base, file):
    back_jar = os.path.join(base, file + '.backup')
    src_jar = os.path.join(base, file)
    
    if not os.path.isfile(src_jar) and not os.path.isfile(back_jar):
        return
    
    if os.path.isfile(back_jar):
        if os.path.isfile(src_jar):
            os.remove(src_jar)
        shutil.copy(back_jar, src_jar)
    else:
        shutil.copy(src_jar, back_jar)
        
def post_decompile(mcp_dir, fml_dir):
    bin_dir = os.path.join(mcp_dir, 'jars', 'bin')
    back_jar = os.path.join(bin_dir, 'minecraft.jar.backup')
    src_jar = os.path.join(bin_dir, 'minecraft.jar')
    
    if not os.path.isfile(src_jar):
        return
        
    print 'Stripping META-INF from minecraft.jar'
    
    zip_in = zipfile.ZipFile(back_jar, mode='a')
    zip_out = zipfile.ZipFile(src_jar, 'w', zipfile.ZIP_DEFLATED)
    for i in zip_in.filelist:
        if not i.filename.startswith('META-INF'):
            c = zip_in.read(i.filename)
            zip_out.writestr(i.filename, c)
        else:
            print 'Skipping: %s' % i.filename
    zip_out.close()
    
def reset_logger():
    log = logging.getLogger()
    while len(log.handlers) > 0:
        log.removeHandler(log.handlers[0])
        
count = 0
def cleanup_source(path):
    path = os.path.normpath(path)
    regex_cases_before = re.compile(r'((case|default).+\r?\n)\r?\n', re.MULTILINE) #Fixes newline after case before case body
    regex_cases_after = re.compile(r'\r?\n(\r?\n[ \t]+(case|default))', re.MULTILINE) #Fixes newline after case body before new case
    
    def updatefile(src_file):
        global count
        tmp_file = src_file + '.tmp'
        count = 0
        with open(src_file, 'r') as fh:
            buf = fh.read()
            
        def fix_cases(match):
            global count
            count += 1
            return match.group(1)
            
        buf = regex_cases_before.sub(fix_cases, buf)
        buf = regex_cases_after.sub(fix_cases, buf)
        if count > 0:
            with open(tmp_file, 'w') as fh:
                fh.write(buf)
            shutil.move(tmp_file, src_file)
            
    for path, _, filelist in os.walk(path, followlinks=True):
        sub_dir = os.path.relpath(path, path)
        for cur_file in fnmatch.filter(filelist, '*.java'):
            src_file = os.path.normpath(os.path.join(path, cur_file))
            updatefile(src_file)
        
def setup_fml(fml_dir, mcp_dir):
    sys.path.append(mcp_dir)
    from runtime.decompile import decompile
    from runtime.cleanup import cleanup
    from runtime.commands import Commands, CLIENT, SERVER
    
    src_dir = os.path.join(mcp_dir, 'src')
        
    if os.path.isdir(src_dir):
        os.chdir(mcp_dir)
        cleanup(None, False)
        reset_logger()
        os.chdir(fml_dir)
        
    if os.path.isdir(src_dir):
        print 'Please make sure to backup your modified files, and say yes when it asks you to do cleanup.'
        sys.exit(1)
    
    #download fernflower/argo/asm/guava
    if not download_deps(mcp_dir):
        sys.exit(1)
    
    def applyrg_shunt(self, side, reobf=False, applyrg_real = Commands.applyrg):
        jars = {CLIENT: self.jarclient, SERVER: self.jarserver}
        
        binDir = os.path.join(fml_dir, 'bin')
        if not os.path.isdir(binDir):
            os.makedirs(binDir)
            
        
        if (side == CLIENT):        
            #Compile AccessTransformer
            self.logger.info('> Compiling AccessTransformer')
            forkcmd = ('%s -Xlint:-options -deprecation -g -source 1.6 -target 1.6 -classpath "{classpath}" -sourcepath "{sourcepath}" -d "{outpath}" "{target}"' % self.cmdjavac).format(
                classpath=os.pathsep.join(['.', os.path.join(mcp_dir, 'lib', '*')]), sourcepath=os.path.join(fml_dir, 'common'), outpath=os.path.join(fml_dir, 'bin'), 
                target=os.path.join(fml_dir, 'common', 'cpw', 'mods', 'fml', 'common', 'asm', 'transformers', 'AccessTransformer.java'))
                
            if not runcmd(self, forkcmd, echo=False):
                sys.exit(1)
                
            self.logger.info('> Compiling MCPMerger')
            forkcmd = ('%s -Xlint:-options -deprecation -g -source 1.6 -target 1.6 -classpath "{classpath}" -sourcepath "{sourcepath}" -d "{outpath}" "{target}"' % self.cmdjavac).format(
                classpath=os.pathsep.join(['.', os.path.join(mcp_dir, 'lib', '*')]), sourcepath=os.path.join(fml_dir, 'common'), outpath=os.path.join(fml_dir, 'bin'), 
                target=os.path.join(fml_dir, 'common', 'cpw', 'mods', 'fml', 'common', 'asm', 'transformers', 'MCPMerger.java'))
                
            if not runcmd(self, forkcmd, echo=False):
                sys.exit(1)
            
            self.logger.info('> Running MCPMerger')
            #Run MCPMerger
            forkcmd = ('%s -classpath "{classpath}" cpw.mods.fml.common.asm.transformers.MCPMerger "{mergecfg}" "{client}" "{server}"' % self.cmdjava).format(
                classpath=os.pathsep.join([os.path.join(mcp_dir, 'lib', '*'), binDir]), mergecfg=os.path.join(fml_dir, 'mcp_merge.cfg'), client=jars[CLIENT], server=jars[SERVER])
                
            if not runcmd(self, forkcmd):
                sys.exit(1)
        
        self.logger.info('> Running AccessTransformer')
        #Run AccessTransformer
        forkcmd = ('%s -classpath "{classpath}" cpw.mods.fml.common.asm.transformers.AccessTransformer "{jar}" "{fmlconfig}"' % self.cmdjava).format(
            classpath=os.pathsep.join([os.path.join(mcp_dir, 'lib', '*'), binDir]), jar=jars[side], fmlconfig=os.path.join(fml_dir, 'common', 'fml_at.cfg'))
            
        forge_cfg = os.path.join(mcp_dir, 'forge', 'common', 'forge_at.cfg')
        if os.path.isfile(forge_cfg):
            self.logger.info('   Forge config detected')
            forkcmd += ' "%s"' % forge_cfg

        for dirname, dirnames, filenames in os.walk(os.path.join(mcp_dir, 'forge', 'accesstransformers')):
            for filename in filenames:
                accesstransformer = os.path.join(dirname, filename)
                if os.path.isfile(accesstransformer):              
                    self.logger.info('   Access Transformer "%s" detected' % filename)
                    forkcmd += ' "%s"' % accesstransformer
        
        if not runcmd(self, forkcmd):
            sys.exit(1)
        
        self.logger.info('> Really Applying Retroguard')
        applyrg_real(self, side, reobf)
    
    #Check the original jars not the transformed jars
    def checkjars_shunt(self, side, checkjars_real = Commands.checkjars):
        self.jarclient = self.jarclient + '.backup'
        self.jarserver = self.jarserver + '.backup'
        #print 'Jar Check %s %s %s' % (side, self.jarclient, self.jarserver)
        ret = checkjars_real(self, side)
        self.jarclient = self.jarclient[:-7]
        self.jarserver = self.jarserver[:-7]
        #print 'Jar Check out %s %s %s' % (side, self.jarclient, self.jarserver)
        return ret
    
    try:
        pre_decompile(mcp_dir, fml_dir)
        
        os.chdir(mcp_dir)
        Commands.applyrg = applyrg_shunt
        Commands.checkjars = checkjars_shunt
        #decompile -d -n -r
        #         Conf  JAD    CSV    -r    -d    -a     -n    -p     -o     -l     -g     -c     -s
        decompile(None, False, False, True, True, False, True, False, False, False, False, False, False)
        reset_logger()
        os.chdir(fml_dir)
        
        post_decompile(mcp_dir, fml_dir)
        
    except SystemExit, e:
        print 'Decompile Exception: %d ' % e.code
        raise e   

    if not os.path.isdir(src_dir):
        print 'Something went wrong, src folder not found at: %s' % src_dir
        sys.exit(1)
        
    #cleanup_source
    cleanup_source(src_dir)
    
    merge_client_server(mcp_dir)
    
    os.chdir(mcp_dir)
    commands = Commands(verify=True)
    updatemd5_side(mcp_dir, commands, CLIENT)
    updatemd5_side(mcp_dir, commands, SERVER)
    reset_logger()
        
    os.chdir(fml_dir)
    
def updatemd5_side(mcp_dir, commands, side):
    sys.path.append(mcp_dir)
    from runtime.mcp import recompile_side, updatemd5_side
    from runtime.commands import SIDE_NAME
    
    recomp = recompile_side(commands, side)
    if recomp:
        commands.logger.info('> Generating %s md5s', SIDE_NAME[side])
        commands.gathermd5s(side, skip_fml=True)

def cmdsplit(args):
    if os.sep == '\\':
        args = args.replace('\\', '\\\\')
    return shlex.split(args)
    
def runcmd(commands, command, echo=True):
    forklist = cmdsplit(command)
    process = subprocess.Popen(forklist, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1)
    output, _ = process.communicate()    
    
    if echo:
        for line in output.splitlines():
            commands.logger.info(line)

    if process.returncode:
        if not echo:        
            for line in output.splitlines():
                commands.logger.info(line)
        return False
    return True

def merge_client_server(mcp_dir):
    client = os.path.join(mcp_dir, 'src', 'minecraft')
    server = os.path.join(mcp_dir, 'src', 'minecraft_server')
    shared = os.path.join(mcp_dir, 'src', 'common')
    
    if not os.path.isdir(shared):
        os.makedirs(shared)
    
    if not os.path.isdir(client) or not os.path.isdir(server):
        return
        
    if not os.path.isdir(shared):
        os.makedirs(shared)
        
    #Nasty hack, but these three files sometimes decompile differently, but are identical, so just take the client file
    special_cases = ['GuiStatsComponent.java', 'HttpUtilRunnable.java', 'PlayerUsageSnooper.java', 'RConThreadClient.java']
        
    for path, _, filelist in os.walk(client, followlinks=True):
        for cur_file in filelist:
            f_client = os.path.normpath(os.path.join(client, path[len(client)+1:], cur_file)).replace(os.path.sep, '/')
            f_server = os.path.normpath(os.path.join(server, path[len(client)+1:], cur_file)).replace(os.path.sep, '/')
            f_shared = os.path.normpath(os.path.join(shared, path[len(client)+1:], cur_file)).replace(os.path.sep, '/')
            
            if not os.path.isfile(f_client) or not os.path.isfile(f_server):
                continue
                
            md5_c = ""
            md5_s = ""
            with open(f_client, 'rb') as fh:
                md5_c = md5(fh.read()).hexdigest()
            with open(f_server, 'rb') as fh:
                md5_s = md5(fh.read()).hexdigest()
                
            if md5_c != md5_s and not cur_file in special_cases:
                continue
                
            new_dir = os.path.join(shared, path[len(client)+1:])
            if not os.path.isdir(new_dir):
                os.makedirs(new_dir)
                
            shutil.move(f_client, f_shared)
            os.remove(f_server)
            
    cleanDirs(server)
    cleanDirs(client)

def apply_fml_patches(fml_dir, mcp_dir, src_dir, copy_files=True):
    #Delete /common/cpw to get rid of the Side/SideOnly classes used in decompilation
    cpw_dir = os.path.join(src_dir, 'common', 'cpw')
    print 'Deleting common/cpw: ' + cpw_dir
    if os.path.isdir(cpw_dir):
        shutil.rmtree(cpw_dir)
        
    #patch files
    print 'Applying Forge ModLoader patches'
    sys.stdout.flush()

    if os.path.isdir(os.path.join(fml_dir, 'patches', 'minecraft')):
        apply_patches(mcp_dir, os.path.join(fml_dir, 'patches', 'minecraft'), src_dir)
    if copy_files and os.path.isdir(os.path.join(fml_dir, 'client')):
        copytree(os.path.join(fml_dir, 'client'), os.path.join(src_dir, 'minecraft'))

    #delete argo
    if os.path.isdir(os.path.join(src_dir, 'minecraft', 'argo')):
        shutil.rmtree(os.path.join(src_dir, 'minecraft', 'argo'))
            
    if os.path.isdir(os.path.join(fml_dir, 'patches', 'common')):
        apply_patches(mcp_dir, os.path.join(fml_dir, 'patches', 'common'), src_dir)
    if copy_files and os.path.isdir(os.path.join(fml_dir, 'common')):
        copytree(os.path.join(fml_dir, 'common'), os.path.join(src_dir, 'common'))

def finish_setup_fml(fml_dir, mcp_dir):
    sys.path.append(mcp_dir)
    from runtime.updatenames import updatenames
    from runtime.updatemd5 import updatemd5
    from runtime.updatemcp import updatemcp
    
    os.chdir(mcp_dir)
    updatenames(None, True, False, False)
    reset_logger()
    updatemd5(None, True, False, False)
    reset_logger()
    os.chdir(fml_dir)

def apply_patches(mcp_dir, patch_dir, target_dir, find=None, rep=None):
    sys.path.append(mcp_dir)
    from runtime.commands import cmdsplit
    
    temp = os.path.abspath('temp.patch')
    cmd = cmdsplit('patch -p2 -i "%s" ' % temp)
    
    if os.name == 'nt':
        applydiff = os.path.abspath(os.path.join(mcp_dir, 'runtime', 'bin', 'applydiff.exe'))
        cmd = cmdsplit('"%s" -uf -p2 -i "%s"' % (applydiff, temp))
    
    for path, _, filelist in os.walk(patch_dir, followlinks=True):
        for cur_file in fnmatch.filter(filelist, '*.patch'):
            patch_file = os.path.normpath(os.path.join(patch_dir, path[len(patch_dir)+1:], cur_file))
            target_file = os.path.join(target_dir, fix_patch(patch_file, temp, find, rep))
            process = subprocess.Popen(cmd, cwd=target_dir, bufsize=-1)
            process.communicate()

    if os.path.isfile(temp):
        os.remove(temp)

def fix_patch(in_file, out_file, find=None, rep=None):
    in_file = os.path.normpath(in_file)
    if out_file is None:
        tmp_filename = in_file + '.tmp'
    else:
        out_file = os.path.normpath(out_file)
        tmp_file = out_file
        dir_name = os.path.dirname(out_file)
        if dir_name:
            if not os.path.exists(dir_name):
                os.makedirs(dir_name)
    file = 'not found'
    with open(in_file, 'rb') as inpatch:
        with open(tmp_file, 'wb') as outpatch:
            for line in inpatch:
                line = line.rstrip('\r\n')
                if line[:3] in ['+++', '---', 'Onl', 'dif']:
                    if not find == None and not rep == None:
                        line = line.replace('\\', '/').replace(find, rep).replace('/', os.sep)
                    else:
                        line = line.replace('\\', '/').replace('/', os.sep)
                    outpatch.write(line + os.linesep)
                else:
                    outpatch.write(line + os.linesep)
                if line[:3] == '---':
                    file = line[line.find(os.sep, line.find(os.sep)+1)+1:]
                    
    if out_file is None:
        shutil.move(tmp_file, in_file)
    return file
        
#Taken from: http://stackoverflow.com/questions/7545299/distutil-shutil-copytree
def _mkdir(newdir):
    """works the way a good mkdir should :)
        - already exists, silently complete
        - regular file in the way, raise an exception
        - parent directory(ies) does not exist, make them as well
    """
    if os.path.isdir(newdir):
        pass
    elif os.path.isfile(newdir):
        raise OSError("a file with the same name as the desired " \
                      "dir, '%s', already exists." % newdir)
    else:
        head, tail = os.path.split(newdir)
        if head and not os.path.isdir(head):
            _mkdir(head)
        #print "_mkdir %s" % repr(newdir)
        if tail:
            os.mkdir(newdir)
            
#Taken from: http://stackoverflow.com/questions/7545299/distutil-shutil-copytree
def copytree(src, dst, verbose=0, symlinks=False):
    """Recursively copy a directory tree using copy2().

    The destination directory must not already exist.
    If exception(s) occur, an Error is raised with a list of reasons.

    If the optional symlinks flag is true, symbolic links in the
    source tree result in symbolic links in the destination tree; if
    it is false, the contents of the files pointed to by symbolic
    links are copied.

    XXX Consider this example code rather than the ultimate tool.

    """
    
    if verbose == -1:
        verbose = len(os.path.abspath(dst)) + 1
    names = os.listdir(src)
    # os.makedirs(dst)
    _mkdir(dst) # XXX
    errors = []
    for name in names:
        srcname = os.path.join(src, name)
        dstname = os.path.join(dst, name)
        try:
            if symlinks and os.path.islink(srcname):
                linkto = os.readlink(srcname)
                os.symlink(linkto, dstname)
            elif os.path.isdir(srcname):
                copytree(srcname, dstname, verbose, symlinks)
            else:
                shutil.copy2(srcname, dstname)
                if verbose > 0:
                    print os.path.abspath(dstname)[verbose:]
            # XXX What about devices, sockets etc.?
        except (IOError, os.error), why:
            errors.append((srcname, dstname, str(why)))
        # catch the Error from the recursive copytree so that we can
        # continue with other files
        except Exception, err:
            errors.extend(err.args[0])
    try:
        shutil.copystat(src, dst)
    except WindowsError:
        # can't copy file access times on Windows
        pass
        
def cleanDirs(path):
    if not os.path.isdir(path):
        return
 
    files = os.listdir(path)
    if len(files):
        for f in files:
            fullpath = os.path.join(path, f)
            if os.path.isdir(fullpath):
                cleanDirs(fullpath)
 
    files = os.listdir(path)
    if len(files) == 0:
        os.rmdir(path)
        
def merge_tree(root_src_dir, root_dst_dir):
    for src_dir, dirs, files in os.walk(root_src_dir):
        dst_dir = src_dir.replace(root_src_dir, root_dst_dir)
        if not os.path.exists(dst_dir):
            os.mkdir(dst_dir)
        for file_ in files:
            src_file = os.path.join(src_dir, file_)
            dst_file = os.path.join(dst_dir, file_)
            if os.path.exists(dst_file):
                os.remove(dst_file)
            shutil.copy(src_file, dst_dir)
            
def setup_mcp(fml_dir, mcp_dir, dont_gen_conf=True):
    backup = os.path.join(mcp_dir, 'runtime', 'commands.py.bck')
    runtime = os.path.join(mcp_dir, 'runtime', 'commands.py')
    patch = os.path.join(fml_dir, 'commands.patch')
    
    print 'Setting up MCP'
    if os.path.isfile(backup):
        print '> Restoring commands.py backup'
        if os.path.exists(runtime):
            os.remove(runtime)
        shutil.copy(backup, runtime)
    else:
        print '> Backing up commands.py'
        shutil.copy(runtime, backup)
    
    if not os.path.isfile(patch):
        raise Exception('Commands.py patch not found %s' % patch)
        return
        
    temp = os.path.abspath('temp.patch')
    cmd = 'patch -i "%s" ' % temp
    
    windows = os.name == 'nt'
    if windows:
        applydiff = os.path.abspath(os.path.join(mcp_dir, 'runtime', 'bin', 'applydiff.exe'))
        cmd = '"%s" -uf -i "%s"' % (applydiff, temp)
        
    if os.sep == '\\':
        cmd = cmd.replace('\\', '\\\\')
    cmd = shlex.split(cmd)
    
    if windows:
        print 'Patching file %s' % os.path.normpath(runtime)
    fix_patch(patch, temp)
    process = subprocess.Popen(cmd, cwd=os.path.join(mcp_dir, 'runtime'), bufsize=-1)
    process.communicate()

    if os.path.isfile(temp):
        os.remove(temp)
    
    try:
        sys.path.append(mcp_dir)
        from runtime.commands import commands_sanity_check
        commands_sanity_check()
    except ImportError as ex:
        print 'Could not verify commands.py patch integrity, this typically means that you are not in a clean MCP environment.'
        print 'Download a clean version of MCP 7.17 and try again'
        print ex
        sys.exit(1)
    
    mcp_conf = os.path.join(mcp_dir, 'conf')
    mcp_conf_bak = os.path.join(mcp_dir, 'conf.bak')
    fml_conf = os.path.join(fml_dir, 'conf')
    
    if not dont_gen_conf:
        if os.path.isdir(mcp_conf_bak):
            print 'Reverting old conf backup folder'
            shutil.rmtree(mcp_conf)
            os.rename(mcp_conf_bak, mcp_conf)

        get_conf_copy(mcp_dir, fml_dir)

        print 'Backing up MCP Conf'
        os.rename(mcp_conf, mcp_conf_bak)
    else:
        shutil.rmtree(mcp_conf)
    
    print 'Copying FML conf'
    shutil.copytree(fml_conf, mcp_conf)
    
    #update workspace
    print 'Fixing MCP Workspace'
    if not os.path.isdir(os.path.join(fml_dir, 'eclipse', 'Clean-Client')):
        mcp_eclipse = os.path.join(mcp_dir, 'eclipse')
        if os.path.isdir(mcp_eclipse):
            shutil.rmtree(mcp_eclipse)
        shutil.copytree(os.path.join(fml_dir, 'eclipse'), mcp_eclipse)

def normaliselines(in_filename):
    in_filename = os.path.normpath(in_filename)
    tmp_filename = in_filename + '.tmp'

    with open(in_filename, 'rb') as in_file:
        with open(tmp_filename, 'wb') as out_file:
            out_file.write(in_file.read().replace('\r\n', '\n'))

    shutil.move(tmp_filename, in_filename)

def get_conf_copy(mcp_dir, fml_dir):
    #Lets grab the files we dont work on
    for file in ['astyle.cfg', 'version.cfg', 'newids.csv']:
        dst_file = os.path.normpath(os.path.join(fml_dir, 'conf', file))
        src_file = os.path.normpath(os.path.join(mcp_dir, 'conf', file))
        if not os.path.isdir(os.path.dirname(dst_file)):
            os.makedirs(os.path.dirname(dst_file))
        if os.path.exists(dst_file):
            os.remove(dst_file)
        shutil.copy(src_file, dst_file)
        normaliselines(dst_file)
        print 'Grabbing: ' + src_file

    common_srg = gen_merged_srg(mcp_dir, fml_dir)
    common_exc = gen_merged_exc(mcp_dir, fml_dir)
    common_map = gen_shared_searge_names(common_srg, common_exc)
    #ToDo use common_map to merge the remaining csvs, client taking precidense, setting the common items to side '2' and editing commands.py in FML to have 'if csv_side == side || csv_side == '2''
    gen_merged_csv(common_map, os.path.join(mcp_dir, 'conf', 'fields.csv'), os.path.join(fml_dir, 'conf', 'fields.csv'))
    gen_merged_csv(common_map, os.path.join(mcp_dir, 'conf', 'methods.csv'), os.path.join(fml_dir, 'conf', 'methods.csv'))
    gen_merged_csv(common_map, os.path.join(mcp_dir, 'conf', 'params.csv'), os.path.join(fml_dir, 'conf', 'params.csv'), main_key='param')
    
def gen_merged_srg(mcp_dir, fml_dir):
    print 'Generating merged Retroguard data'
    srg_client = os.path.join(mcp_dir, 'conf', 'client.srg')
    srg_server = os.path.join(mcp_dir, 'conf', 'server.srg')
    
    if not os.path.isfile(srg_client) or not os.path.isfile(srg_server):
        print 'Could not find client and server srg files in "%s"' % mcp_dir
        return False
        
    client = {'PK:': {}, 'CL:': {}, 'FD:': {}, 'MD:': {}}
    with open(srg_client, 'r') as fh:
        for line in fh:
            pts = line.rstrip('\r\n').split(' ')
            if pts[0] == 'MD:':
                client[pts[0]][pts[1] + ' ' + pts[2]] = pts[3] + ' ' + pts[4]
            else:
                client[pts[0]][pts[1]] = pts[2]
    
    server = {'PK:': {}, 'CL:': {}, 'FD:': {}, 'MD:': {}}
    with open(srg_server, 'r') as fh:
        for line in fh:
            pts = line.rstrip('\r\n').split(' ')
            if pts[0] == 'MD:':
                server[pts[0]][pts[1] + ' ' + pts[2]] = pts[3] + ' ' + pts[4]
            else:
                server[pts[0]][pts[1]] = pts[2]
    
    common = {'PK:': {}, 'CL:': {}, 'FD:': {}, 'MD:': {}}
    for type in common:
        for key, value in client[type].items():
            if key in server[type]:
                if value == server[type][key]:
                    client[type].pop(key)
                    server[type].pop(key)
                    common[type][key] = value

    for type in common:
        for key, value in client[type].items():
            common[type][key] = value #+ ' #C'
            
    for type in common:
        for key, value in server[type].items():
            common[type][key] = value #+ ' #S'
            
    #Print joined retroguard files
    with open(os.path.join(fml_dir, 'conf', 'joined.srg'), 'wb') as f:
        for type in ['PK:', 'CL:', 'FD:', 'MD:']:
            for key in sorted(common[type]):
                f.write('%s %s %s\n' % (type, key, common[type][key]))
    
    return common

def gen_merged_exc(mcp_dir, fml_dir):
    print 'Reading merged MCInjector config'
    exc_joined = os.path.join(mcp_dir, 'conf', 'joined.exc')
    
    joined = {}
    with open(exc_joined, 'r') as fh:
        for line in fh:
            if not line.startswith('#'):
                pts = line.rstrip('\r\n').split('=')
                joined[pts[0]] = pts[1]
    
    return joined

def gen_shared_searge_names(common_srg, common_exc):
    field = re.compile(r'field_[0-9]+_[a-zA-Z_]+$')
    method = re.compile(r'func_[0-9]+_[a-zA-Z_]+')
    param = re.compile(r'p_[\w]+_\d+_')
    
    print 'Gathering list of common searge names'
    
    searge = []
    
    for key, value in common_srg['FD:'].items():
        m = field.search(value)
        if not m is None:
            if not m.group(0) in searge:
                searge.append(m.group(0))
                
    for key, value in common_srg['MD:'].items():
        m = method.search(value)
        if not m is None and not '#' in value:
            if not m.group(0) in searge:
                searge.append(m.group(0))
                
    for key, value in common_exc.items():
        m = param.findall(value)
        if not m is None:
            for p in m:
                if not p in searge:
                    searge.append(p)
                    
    return searge

def gen_merged_csv(common_map, in_file, out_file, main_key='searge'):
    reader = csv.DictReader(open(in_file, 'r'))
    print 'Generating merged csv for %s' % os.path.basename(in_file)
    common = []
    added = []
    for row in reader:
        if not row[main_key] in added:
            row['side'] = '2'
            common.append(row)
            added.append(row[main_key])
    writer = csv.DictWriter(open(out_file, 'wb'), fieldnames=reader.fieldnames, lineterminator='\n')
    writer.writeheader()
    for row in sorted(common, key=lambda row: row[main_key]):
        writer.writerow(row)