import os, os.path, sys import urllib, zipfile, json, urllib2 import shutil, glob, fnmatch import subprocess, logging, re, shlex import csv, ConfigParser from hashlib import md5 # pylint: disable-msg=E0611 from pprint import pprint from zipfile import ZipFile from pprint import pprint from contextlib import closing #========================================================================== # Utility Functions #========================================================================== def config_get_section(config, section): dict = {} options = config.options(section) for option in options: try: dict[option] = config.get(section, option) except: dict[option] = None return dict def read_mc_versions(fml_dir, version=None, work_dir=None): ########################################################################################################### #Read mc_versions.cfg from the fml folder for the specified version, or the default #version if none is specified. # #Should return a dictionary with the following keys: # 'new_laucher' True if the version uses the new launcher structure for assets/libraries # 'client_url' URL path to the minecraft client jar # 'client_md5' MD5 checksum of the client jar # 'server_url' URL path to the minecraft server jar # 'server_md5' MD5 checksum of the server jar # 'json_url' URL path to the json file for this version, or 'None' if new_launcher is false # 'mcp_ver' Human readable version for MCP to use for this version of Minecraft # 'mcp_url' URL path for MCP to download, or 'None' if there isn't one avalible # 'mcp_md5' MD5 checksum of the MCP archive # 'mcp_file' File path to the MCP archive # # If work_dir is specified the following keys also contain values: # 'natives_dir' File path to native libraries used by Minecraft # 'library_dir' File path to 'libraries' folder # if 'new_launcher' is false this points to work_dir/bin/ and will not be used as maven style folder structure # if 'new_launcher' is true this points to work_dir/libraraies/ and will use the maven style folder structure # 'client_file' File path for the client minecraft.jar # 'server_file' File path to the server minecraft jar # 'json_file' # if 'new_launcher' is false, this is 'None' # if 'new_launcher' is true, this is the path to the version json file on disc. # 'asset_dir' Folder containering all assets/resources used by minecraft # # Note: Because a file is give a path it does NOT mean that it exists! So alway check before using ############################################################################################################################ versions_file = os.path.join(fml_dir, 'mc_versions.cfg') if not os.path.isfile(versions_file): print 'Could not find mc_versions.cfg in FML directory.' sys.exit(1) config = ConfigParser.ConfigParser() config.read(versions_file) default = config_get_section(config, 'default') if version is None: version = default['current_ver'] if not config.has_section(version): print 'Error: Invalid minecraft version, could not find \'%s\' in mc_versions.cfg' % version sys.exit(1) mc_info = config_get_section(config, version) mc_info['downloads'] = [] mc_info['version'] = version if not 'client_url' in mc_info.keys(): mc_info['new_launcher'] = True base_url = 'https://s3.amazonaws.com/Minecraft.Download/versions/%s' % version mc_info['client_url'] = '%s/%s.jar' % (base_url, version) mc_info['json_url'] = '%s/%s.json' % (base_url, version) mc_info['server_url'] = '%s/minecraft_server.%s.jar' % (base_url, version) if not work_dir is None: version_dir = os.path.join(work_dir, 'versions', version) mc_info['natives_dir'] = os.path.join(work_dir, 'libraries', 'natives') mc_info['library_dir'] = os.path.join(work_dir, 'libraries') mc_info['client_file'] = os.path.join(version_dir, '%s.jar' % version) mc_info['json_file'] = os.path.join(version_dir, '%s.json' % version) mc_info['server_file'] = os.path.join(work_dir, 'minecraft_server.%s.jar' % version) mc_info['asset_dir'] = os.path.join(work_dir, 'assets') else: mc_info['new_launcher'] = False mc_info['json_url'] = None if not work_dir is None: mc_info['natives_dir'] = os.path.join(work_dir, 'bin', 'natives') mc_info['library_dir'] = os.path.join(work_dir, 'bin') mc_info['client_file'] = os.path.join(work_dir, 'bin', 'minecraft.jar') mc_info['json_file'] = None mc_info['server_file'] = os.path.join(work_dir, 'minecraft_server.jar') mc_info['asset_dir'] = os.path.join(work_dir, 'resources') for lib in default['libraries'].split(' '): mc_info['libraries'].append({ 'url' : default['base_url'] + lib, 'file' : os.path.join(work_dir, 'bin', lib), 'extract' : None, 'md5' : None }) for native in default['natives'].split(' '): mc_info['libraries'].append({ 'url_ex' : default['base_url'] + lib, 'file' : os.path.join(work_dir, 'bin', lib), 'extract' : { 'exclude' : ['META-INF/'] }, 'md5' : None }) if not mc_info['mcp_url'].startswith('http'): mc_info['mcp_url'] = None mc_info['mcp_file'] = os.path.join(fml_dir, 'mcp%s.zip' % mc_info['mcp_ver']) if not work_dir is None: for x in ['natives_dir', 'library_dir', 'asset_dir']: if not os.path.isdir(mc_info[x]): os.makedirs(mc_info[x]) for x in ['client_file', 'server_file', 'json_file', 'mcp_file']: if mc_info[x] is None: continue dir = os.path.dirname(mc_info[x]) if not os.path.isdir(dir): os.makedirs(dir) return mc_info def download_file(url, target, md5=None, root=None, prefix=''): name = os.path.basename(target) if not root is None: name = os.path.abspath(target) name = name[len(os.path.abspath(root)) + 1:] dir = os.path.dirname(target) if not os.path.isdir(dir): os.makedirs(dir) if os.path.isfile(target) and not md5 == None: if not get_md5(target) == md5: print '%s%s Modified, removing' % (prefix, name) os.remove(target) if not os.path.isfile(target): try: urllib.urlretrieve(url, target) if not md5 == None: if not get_md5(target) == md5: print '%sDownload of %s failed md5 check, deleting' % (prefix, name) os.remove(target) return False if prefix == '': print 'Downloaded %s' % name else: print '%s%s Done' % (prefix, name) except Exception as e: print e print '%sDownload of %s failed, download it manually from \'%s\' to \'%s\'' % (prefix, target, url, target) return False return True def get_headers(url): #Connects to the given URL and requests just the headers, No data #Used when talking to Minecraft's asset/library server to gather server side MD5's #Returns a dictionary of all headers class HeadRequest(urllib2.Request): def get_method(self): return 'HEAD' response = urllib2.urlopen(HeadRequest(url)) array = [line.rstrip('\r\n') for line in response.info().headers] dict = {} for line in array: pts = line.split(':', 1) pts[1] = pts[1].strip() #Strip the first and last "s if the stirng is surrounded with them if pts[1][0] == '"' and pts[1][-1] == '"': pts[1] = pts[1][1:-1] dict[pts[0]] = pts[1] return dict def get_md5(file): #Returns the MD5 digest of the specified file, or None if the file doesnt exist if not os.path.isfile(file): return None with closing(open(file, 'rb')) as fh: return md5(fh.read()).hexdigest() def fix_patch(in_file, out_file, find=None, rep=None): #Fixes the following issues in the patch file if they exist: # Normalizes the path seperators for the current OS # Normalizes the line endings # Returns the path that the file wants to apply to in_file = os.path.normpath(in_file) if out_file is None: tmp_file = in_file + '.tmp' else: out_file = os.path.normpath(out_file) tmp_file = out_file dir_name = os.path.dirname(out_file) if dir_name: if not os.path.exists(dir_name): os.makedirs(dir_name) file = 'not found' with open(in_file, 'rb') as inpatch: with open(tmp_file, 'wb') as outpatch: for line in inpatch: line = line.rstrip('\r\n') if line[:3] in ['+++', '---', 'Onl', 'dif']: if not find == None and not rep == None: line = line.replace('\\', '/').replace(find, rep).replace('/', os.sep) else: line = line.replace('\\', '/').replace('/', os.sep) outpatch.write(line + os.linesep) else: outpatch.write(line + os.linesep) if line[:3] == '---': file = line[line.find(os.sep, line.find(os.sep)+1)+1:] if out_file is None: shutil.move(tmp_file, in_file) return file def apply_patch(patch, target, mcp_dir): temp = os.path.abspath('temp.patch') cmd = 'patch -i "%s" ' % temp if os.name == 'nt': applydiff = os.path.abspath(os.path.join(mcp_dir, 'runtime', 'bin', 'applydiff.exe')) cmd = '"%s" -uf -i "%s"' % (applydiff, temp) if os.sep == '\\': cmd = cmd.replace('\\', '\\\\') cmd = shlex.split(cmd) fix_patch(patch, temp) process = subprocess.Popen(cmd, cwd=os.path.join(mcp_dir, 'runtime'), bufsize=-1) process.communicate() if os.path.isfile(temp): os.remove(temp) def file_backup(file, md5=None): #Takes a backup of the passed in file #Verifying the md5 sum if it's specified #At the end of this things should be in one of two states: # 1) file and file.backup exist, both are valid and match the md5 provided # 2) neither file or file.backup exist, as they both failed the md5 check base = os.path.dirname(file) name = os.path.basename(file) bck = os.path.join(base, name + '.backup') src = os.path.join(base, name) if not os.path.isfile(src) and not os.path.isfile(bck): return if os.path.isfile(bck): if get_md5(bck) == md5 or md5 is None: if os.path.isfile(src): os.remove(src) shutil.move(bck, src) else: os.remove(bck) if os.path.isfile(src): if not get_md5(src) == md5 and not md5 is None: print 'Modified %s detected, removing' % os.path.basename(src_jar) os.remove(src) else: shutil.copy(src, bck) def normaliselines(file): #Normalises the lines of the specified file to linux \n line endings file = os.path.normpath(file) tmp = file + '.tmp' with open(file, 'rb') as in_file: with open(tmp, 'wb') as out_file: out_file.write(in_file.read().replace('\r\n', '\n')) shutil.move(tmp, file) def load_srg(srg_file, reverse=False): #Loads a Retroguard .srg file into a dictonary #If reverse if true, the mappings are reversed with open(srg_file, 'r') as fh: lines = fh.readlines() srg = {'CL:': {}, 'MD:': {}, 'FD:': {}, 'PK:': {}} for line in lines: line = line.strip() if len(line) == 0: continue if line[0] == '#': continue args = line.split(' ') type = args[0] if type == 'PK:' or type == 'CL:' or type == 'FD:': srg[type][args[1]] = args[2] elif type == 'MD:': srg[type][args[1] + ' ' + args[2]] = args[3] + ' ' + args[4] else: assert 'Unknown type %s' % line if reverse: for type,map in srg.items(): srg[type] = dict([[v,k] for k,v in map.items()]) return srg def extract_zip(src, dst, prefix=None, filter=[]): # Extract a zip rchive to the specified folder, # Filtering out anything that matches the supplied filters def is_filtered(name, excludes): for ex in excludes: if name.startswith(ex): return True return name.endswith('/') zip = ZipFile(src) for name in zip.namelist(): if is_filtered(name, filter): continue out_file = os.path.join(dst, os.sep.join(name.split('/'))) if not os.path.isfile(out_file): dir = os.path.dirname(out_file) if not os.path.isdir(dir): os.makedirs(dir) if not prefix is None: print '%sExtracting %s' % (prefix, name) out = open(out_file, 'wb') out.write(zip.read(name)) out.flush() out.close() zip.close() def merge_tree(root_src_dir, root_dst_dir, prefix=None): #Merges the source directory into the dest directory, #will overwrite anything the currently exists for src_dir, dirs, files in os.walk(root_src_dir): dst_dir = src_dir.replace(root_src_dir, root_dst_dir) clean_dir = src_dir.replace(root_src_dir, '')[1:] if not os.path.exists(dst_dir): os.mkdir(dst_dir) for file_ in files: src_file = os.path.join(src_dir, file_) dst_file = os.path.join(dst_dir, file_) if os.path.exists(dst_file): os.remove(dst_file) shutil.copy(src_file, dst_dir) if not prefix is None: print('%s%s%s'% (prefix, clean_dir, file_)) def read_file(file): if not os.path.exists(file): return None buf = None with closing(open(file, 'r')) as fh: buf = fh.read() return buf def runcmd(cmd, echo=True, commands=None): forklist = cmdsplit(cmd) process = subprocess.Popen(forklist, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) output, _ = process.communicate() if echo and not commands is None: for line in output.splitlines(): commands.logger.info(line) if process.returncode: if not echo and not commands is None: for line in output.splitlines(): commands.logger.info(line) return False return True def cmdsplit(args): if os.sep == '\\': args = args.replace('\\', '\\\\') return shlex.split(args) def kill_signatures(jar_file): # Removes everything in a jar file's META-INF folder, typically used to remove signature data dir = os.path.dirname(jar_file) name = os.path.basename(jar_file) tmp_jar = os.path.join(dir, '%s.temp' % name) if not os.path.isfile(jar_file): return if os.path.isfile(tmp_jar): os.remove(tmp_jar) shutil.move(jar_file, tmp_jar) print('Stripping META-INF from %s' % jar_file) with closing(zipfile.ZipFile(tmp_jar, mode='a')) as zip_in: with closing(zipfile.ZipFile(jar_file, 'w', zipfile.ZIP_DEFLATED)) as zip_out: for i in zip_in.filelist: if not i.filename.startswith('META-INF'): c = zip_in.read(i.filename) zip_out.writestr(i.filename, c) else: print(' Skipping: %s' % i.filename) os.remove(tmp_jar) def apply_patches(mcp_dir, patch_dir, target_dir, find=None, rep=None): # Attempts to apply a directory full of patch files onto a target directory. sys.path.append(mcp_dir) temp = os.path.abspath('temp.patch') cmd = cmdsplit('patch -p2 -i "%s" ' % temp) if os.name == 'nt': applydiff = os.path.abspath(os.path.join(mcp_dir, 'runtime', 'bin', 'applydiff.exe')) cmd = cmdsplit('"%s" -uf -p2 -i "%s"' % (applydiff, temp)) for path, _, filelist in os.walk(patch_dir, followlinks=True): for cur_file in fnmatch.filter(filelist, '*.patch'): patch_file = os.path.normpath(os.path.join(patch_dir, path[len(patch_dir)+1:], cur_file)) target_file = os.path.join(target_dir, fix_patch(patch_file, temp, find, rep)) process = subprocess.Popen(cmd, cwd=target_dir, bufsize=-1) process.communicate() if os.path.isfile(temp): os.remove(temp) #Taken from: http://stackoverflow.com/questions/7545299/distutil-shutil-copytree def _mkdir(newdir): """works the way a good mkdir should :) - already exists, silently complete - regular file in the way, raise an exception - parent directory(ies) does not exist, make them as well """ if os.path.isdir(newdir): pass elif os.path.isfile(newdir): raise OSError("a file with the same name as the desired " \ "dir, '%s', already exists." % newdir) else: head, tail = os.path.split(newdir) if head and not os.path.isdir(head): _mkdir(head) #print "_mkdir %s" % repr(newdir) if tail: os.mkdir(newdir) #Taken from: http://stackoverflow.com/questions/7545299/distutil-shutil-copytree def copytree(src, dst, verbose=0, symlinks=False): """Recursively copy a directory tree using copy2(). The destination directory must not already exist. If exception(s) occur, an Error is raised with a list of reasons. If the optional symlinks flag is true, symbolic links in the source tree result in symbolic links in the destination tree; if it is false, the contents of the files pointed to by symbolic links are copied. XXX Consider this example code rather than the ultimate tool. """ if verbose == -1: verbose = len(os.path.abspath(dst)) + 1 names = os.listdir(src) # os.makedirs(dst) _mkdir(dst) # XXX errors = [] for name in names: srcname = os.path.join(src, name) dstname = os.path.join(dst, name) try: if symlinks and os.path.islink(srcname): linkto = os.readlink(srcname) os.symlink(linkto, dstname) elif os.path.isdir(srcname): copytree(srcname, dstname, verbose, symlinks) else: shutil.copy2(srcname, dstname) if verbose > 0: print os.path.abspath(dstname)[verbose:] # XXX What about devices, sockets etc.? except (IOError, os.error), why: errors.append((srcname, dstname, str(why))) # catch the Error from the recursive copytree so that we can # continue with other files except Exception, err: errors.extend(err.args[0]) try: shutil.copystat(src, dst) except WindowsError: # can't copy file access times on Windows pass #========================================================================== # MCP Related Functions #========================================================================== def download_mcp(mcp_dir, fml_dir, version=None): if os.path.isfile(os.path.join(mcp_dir, 'runtime', 'commands.py')): print 'MCP Detected already, not downloading' return True if os.path.isdir(mcp_dir): print 'Old MCP Directory exists, but MCP was not detected, please delete MCP directory at \'%s\'' % mcp_dir sys.exit(1) mc_info = read_mc_versions(fml_dir, version=version) if not download_file(mc_info['mcp_url'], mc_info['mcp_file'], mc_info['mcp_md5']): sys.exit(1) if not os.path.isdir(mcp_dir): _mkdir(mcp_dir) print 'Extracting MCP to \'%s\'' % mcp_dir extract_zip(mc_info['mcp_file'], mcp_dir, filter=['eclipse']) #, prefix=' ') #If we're not on windows, lets set the executable flag on all shell scripts and astyle-osx if os.name != 'nt': for path, _, filelist in os.walk(mcp_dir): for cur_file in fnmatch.filter(filelist, '*.sh'): file_name = os.path.join(path, cur_file) process = subprocess.Popen(cmdsplit('chmod +x "%s"' % file_name), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) output, _ = process.communicate() process = subprocess.Popen(cmdsplit('chmod +x "%s/runtime/bin/astyle-osx"' % mcp_dir), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=-1) output, _ = process.communicate() #As a helper to build servers, or people who don't wish to download all libraris and assets every time. #Copy all data from 'mcp_data' to the mcp directory data_dir = os.path.join(fml_dir, 'mcp_data') if os.path.isdir(data_dir): print 'Moving mcp_data to MCP' merge_tree(data_dir, mcp_dir)##, prefix=' ') return True def setup_mcp(fml_dir, mcp_dir, gen_conf=True): #Modifies MCP to the state FML needs it in for recompile/reobf/etc to work as we intend it. #Does not decompile minecraft in this stage! print('Setting up MCP') runtime = os.path.join(mcp_dir, 'runtime', 'commands.py') file_backup(runtime) patch = os.path.join(fml_dir, 'commands.patch') if not os.path.isfile(patch): raise Exception('Commands.py patch not found %s' % patch) return print('Patching commands.py') apply_patch(patch, runtime, mcp_dir=mcp_dir) try: sys.path.append(mcp_dir) from runtime.commands import commands_sanity_check commands_sanity_check() except ImportError as ex: print('Could not verify commands.py patch integrity, this typically means that you are not in a clean MCP environment.') print('Download a clean version of MCP %s and try again' % mcp_version) print(ex) sys.exit(1) mcp_conf = os.path.join(mcp_dir, 'conf') mcp_conf_bak = os.path.join(mcp_dir, 'conf.bak') fml_conf = os.path.join(fml_dir, 'conf') if gen_conf: if os.path.isdir(mcp_conf_bak): print 'Reverting old conf backup folder' shutil.rmtree(mcp_conf) os.rename(mcp_conf_bak, mcp_conf) create_merged_conf(mcp_dir, fml_dir) print 'Backing up MCP Conf' os.rename(mcp_conf, mcp_conf_bak) else: shutil.rmtree(mcp_conf) print 'Copying FML conf' shutil.copytree(fml_conf, mcp_conf) create_renamed_conf(mcp_dir, fml_dir) #update workspace if not os.path.isfile(os.path.join(fml_dir, 'fmlbuild.properties-sample')): mcp_eclipse = os.path.join(mcp_dir, 'eclipse') if os.path.isdir(os.path.join(mcp_eclipse, 'Client')) and os.path.isdir(os.path.join(mcp_eclipse, 'Server')): shutil.rmtree(mcp_eclipse) if not os.path.isdir(mcp_eclipse) and os.path.isdir(os.path.join(fml_dir, 'eclipse')): print 'Fixing MCP Workspace' copytree(os.path.join(fml_dir, 'eclipse'), mcp_eclipse) def whereis(filename, rootdir): # Snagged from MCP if not os.path.exists(rootdir): return [] #logging.info('> Searching for %s in %s', filename, rootdir) results = [] for path, _, filelist in os.walk(rootdir): if filename in filelist: results.append(path) return results def find_java(): # Snagged from MCP so we can gather this info without setting up it's Command object results = [] if os.name == 'nt': if not results: import _winreg for flag in [_winreg.KEY_WOW64_64KEY, _winreg.KEY_WOW64_32KEY]: try: k = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, r'Software\JavaSoft\Java Development Kit', 0, _winreg.KEY_READ | flag) version, _ = _winreg.QueryValueEx(k, 'CurrentVersion') k.Close() k = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, r'Software\JavaSoft\Java Development Kit\%s' % version, 0, _winreg.KEY_READ | flag) path, _ = _winreg.QueryValueEx(k, 'JavaHome') k.Close() path = os.path.join(str(path), 'bin') if (runcmd('"%s" -version' % os.path.join(path, 'javac'))): results.append(path) except (OSError): pass if not results: if (runcmd('javac -version')): results.append('') if not results and 'ProgramW6432' in os.environ: results.extend(whereis('javac.exe', os.environ['ProgramW6432'])) if not results and 'ProgramFiles' in os.environ: results.extend(whereis('javac.exe', os.environ['ProgramFiles'])) if not results and 'ProgramFiles(x86)' in os.environ: results.extend(whereis('javac.exe', os.environ['ProgramFiles(x86)'])) else: if not results: if (runcmd('javac -version')): results.append('') if not results: results.extend(whereis('javac', '/usr/bin')) if not results: results.extend(whereis('javac', '/usr/local/bin')) if not results: results.extend(whereis('javac', '/opt')) if not results: print('Java JDK is not installed ! Please install java JDK from http://java.oracle.com') sys.exit(1) return { 'javac' : '"%s"' % os.path.join(results[0], 'javac'), 'java' : '"%s"' % os.path.join(results[0], 'java') } #================================================================================== # MCP Conf Merger Code #================================================================================== def create_merged_conf(mcp_dir, fml_dir): print('Creating merged conf') #Creates the merged conf folder from MCP's conf folder to fml_dir/conf #Lets grab the files we dont work on for file in ['version.cfg', 'newids.csv', 'joined.exc']: dst_file = os.path.join(fml_dir, 'conf', file) src_file = os.path.join(mcp_dir, 'conf', file) if not os.path.isdir(os.path.dirname(dst_file)): os.makedirs(os.path.dirname(dst_file)) if os.path.exists(dst_file): os.remove(dst_file) shutil.copy(src_file, dst_file) normaliselines(dst_file) print(' Copying %s' % os.path.normpath(src_file)) print(' Generating merged Retroguard data') common_srg = create_merged_srg(mcp_dir, fml_dir) print(' Reading merged MCInjector config') common_exc = load_merged_exc(mcp_dir, fml_dir) print(' Gathering list of common searge names') common_map = create_shared_searge_names(common_srg, common_exc) for x in [['fields.csv', 'searge'], ['methods.csv', 'searge'], ['params.csv', 'param']]: print(' Generating merged csv for %s' % x[0]) create_merged_csv(common_map, os.path.join(mcp_dir, 'conf', x[0]), os.path.join(fml_dir, 'conf', x[0]), main_key=x[1]) def create_merged_srg(mcp_dir, fml_dir): #Merges two .srg files together to create one master mapping. # When issues are encountered, they are reported but the client srg is trusted over the server client_file = os.path.join(mcp_dir, 'conf', 'client.srg') server_file = os.path.join(mcp_dir, 'conf', 'server.srg') if not os.path.isfile(client_file) or not os.path.isfile(server_file): print(' Could not find client and server srg files in "%s"' % mcp_dir) return False client = load_srg(client_file) server = load_srg(server_file) common = {'PK:': {}, 'CL:': {}, 'FD:': {}, 'MD:': {}} for type in common: for key, value in client[type].items(): if key in server[type]: if value == server[type][key]: client[type].pop(key) server[type].pop(key) common[type][key] = value for type in common: for key, value in client[type].items(): common[type][key] = value + ' #C' # Tag the client only entries for type in common: for key, value in server[type].items(): common[type][key] = value + ' #S' # Tag the server only entries if not fml_dir is None: #Print joined retroguard files with open(os.path.join(fml_dir, 'conf', 'joined.srg'), 'wb') as f: for type in ['PK:', 'CL:', 'FD:', 'MD:']: for key in sorted(common[type]): f.write('%s %s %s\n' % (type, key, common[type][key])) return common def load_merged_exc(mcp_dir, fml_dir): #Reads the exc file into a dictionary joined = {} with open(os.path.join(mcp_dir, 'conf', 'joined.exc'), 'r') as fh: for line in fh: if not line.startswith('#'): pts = line.rstrip('\r\n').split('=') joined[pts[0]] = pts[1] return joined def create_shared_searge_names(common_srg, common_exc): #Creates an array of all srg names that are common on both the client and server field = re.compile(r'field_[0-9]+_[a-zA-Z_]+$') method = re.compile(r'func_[0-9]+_[a-zA-Z_]+') param = re.compile(r'p_[\w]+_\d+_') searge = [] for key, value in common_srg['FD:'].items(): m = field.search(value) if not m is None and not '#' in value: if not m.group(0) in searge: searge.append(m.group(0)) for key, value in common_srg['MD:'].items(): m = method.search(value) if not m is None and not '#' in value: if not m.group(0) in searge: searge.append(m.group(0)) for key, value in common_exc.items(): m = param.findall(value) if not m is None: for p in m: if not p in searge: searge.append(p) return searge def create_merged_csv(common_map, in_file, out_file, main_key='searge'): #Filter throw the csv and condense 'shared' mappings into one entry of side 2 fields = [] data = [] with closing(open(in_file, 'r')) as fh: reader = csv.DictReader(fh) fields = reader.fieldnames data = [r for r in reader] side = [ [r for r in data if r['side'] == '0' and not r[main_key] in common_map], [r for r in data if r['side'] == '1' and not r[main_key] in common_map], sorted([r for r in data if r[main_key] in common_map], key=lambda row: row['side']) ] added = [] common = [] for row in side[2]: if not row[main_key] in added: row['side'] = '2' added.append(row[main_key]) common.append(row) with closing(open(out_file, 'wb')) as fh: writer = csv.DictWriter(fh, fieldnames=fields, lineterminator='\n') writer.writeheader() for row in sorted(side[0] + side[1] + common, key=lambda row: row[main_key]): writer.writerow(row) def create_renamed_conf(mcp_dir, fml_dir): # Creates copies of the joined srg and exec files with the new Packaged names # Also updates the patches in the conf folder for the new packaged names print('Creating Repackaged data') pkg_file = os.path.join(fml_dir, 'conf', 'packages.csv') pkgs = {} if os.path.isfile(pkg_file): with closing(open(pkg_file)) as fh: reader = csv.DictReader(fh) for line in reader: pkgs[line['class']] = line['package'] def repackage_class(pkgs, cls): if cls.startswith('net/minecraft/src/'): tmp = cls[18:] if tmp in pkgs.keys(): return '%s/%s' % (pkgs[tmp], tmp) return cls for ext in ['srg', 'exc']: regnms = re.compile(r'net/minecraft/src/(\w+)') print(' Creating re-packaged %s' % ext) buf = read_file(os.path.join(mcp_dir, 'conf', 'joined.%s' % ext)) def mapname(match): return repackage_class(pkgs, match.group(0)) buf = regnms.sub(mapname, buf) with closing(open(os.path.join(mcp_dir, 'conf', 'packaged.%s' % ext), 'wb')) as outf: outf.write(buf) print(' Creating re-packaged MCP patches') def fix_patches(patch_in, patch_tmp): regnms = re.compile(r'net\\minecraft\\src\\(\w+)') with closing(open(patch_in, 'r')) as fh: buf = fh.read() def mapname(match): return repackage_class(pkgs, match.group(0).replace('\\', '/')).replace('/', '\\') buf = regnms.sub(mapname, buf) with closing(open(patch_tmp, 'w')) as fh: fh.write(buf) shutil.move(patch_tmp, patch_in) patch_dir = os.path.join(mcp_dir, 'conf', 'patches') fix_patches(os.path.join(patch_dir, 'minecraft_ff.patch' ), os.path.join(patch_dir, 'tmp.patch')) fix_patches(os.path.join(patch_dir, 'minecraft_server_ff.patch'), os.path.join(patch_dir, 'tmp.patch')) #========================================================================== # MCP Decompile Process #========================================================================== def reset_logger(): # Resets the logging handlers, if we don't do this, we get multi-prints from MCP log = logging.getLogger() while len(log.handlers) > 0: log.removeHandler(log.handlers[0]) count = 0 def cleanup_source(path): # We cleanup various things in MCP such as: # astyle differences: # newline after case before case body # newline after case body before new case # We also assign jad-style names to local variables in decompiled code. from rename_vars import rename_class path = os.path.normpath(path) regex_cases_before = re.compile(r'((case|default).+\r?\n)\r?\n', re.MULTILINE) #Fixes newline after case before case body regex_cases_after = re.compile(r'\r?\n(\r?\n[ \t]+(case|default))', re.MULTILINE) #Fixes newline after case body before new case def updatefile(src_file): global count tmp_file = src_file + '.tmp' count = 0 with open(src_file, 'r') as fh: buf = fh.read() def fix_cases(match): global count count += 1 return match.group(1) buf = regex_cases_before.sub(fix_cases, buf) buf = regex_cases_after.sub(fix_cases, buf) old = buf.replace('\r', '') buf = rename_class(old, MCP=True) if count > 0 or buf != old: with open(tmp_file, 'w') as fh: fh.write(buf) shutil.move(tmp_file, src_file) for path, _, filelist in os.walk(path, followlinks=True): sub_dir = os.path.relpath(path, path) for cur_file in fnmatch.filter(filelist, '*.java'): src_file = os.path.normpath(os.path.join(path, cur_file)) updatefile(src_file) compile_tools = True client_jar = None def decompile_minecraft(fml_dir, mcp_dir, disable_at=False, disable_merge=False, enable_server=False, disable_client=False, disable_assets=False): # This is where the bulk of the decompile actually happens # Its a bit of a pain as we need to hook into MCP's Commands class to inject our transformers at certian times. global compile_tools global client_jar sys.path.append(mcp_dir) from runtime.decompile import decompile from runtime.cleanup import cleanup from runtime.commands import Commands, CLIENT, SERVER fml_dir = os.path.abspath(fml_dir) # Make sure the src directroy is dead src_dir = os.path.join(mcp_dir, 'src') if os.path.isdir(src_dir): os.chdir(mcp_dir) cleanup(None, False) reset_logger() os.chdir(fml_dir) if os.path.isdir(src_dir): print 'Please make sure to backup your modified files, and say yes when it asks you to do cleanup.' sys.exit(1) compile_tools = True # Inject ourselves before RetroGuard if it's used def applyrg_shunt(self, side, reobf=False, applyrg_real=Commands.applyrg): transformers_hook(self, side) self.logger.info('> Really Applying Retroguard') applyrg_real(self, side, reobf) # Inject ourselves before SpecialSource if it's used def applyss_shunt(self, side, reobf=False, srg_names=False, in_jar=None, out_jar=None, keep_lvt=False, keep_generics=False, applyss_real=Commands.applyss): transformers_hook(self, side) self.logger.info('> Really Applying SpecialSource') applyss_real(self, side, reobf=reobf, srg_names=srg_names, in_jar=in_jar, out_jar=out_jar, keep_lvt=keep_lvt, keep_generics=keep_generics) # This is our pre-everything hook. # We do the following: # Verify if astyle is installed/accessible, if not then exit # We also compile and run our transformers: # MCPMerge, This merges the runnable code from the server and client creating a single codebase that is complete. # AccessTransformer: # This changes access levels of classes fields and methods based on a configuration map. # Allows up to keep out base edits down because we don't have to patch the access levels of everything. def transformers_hook(self, side): global compile_tools if not self.has_wine and not self.has_astyle: self.logger.error('!! Please install either wine or astyle for source cleanup !!') self.logger.error('!! This is REQUIRED by FML/Forge Cannot proceed !!') sys.exit(1) jars = {CLIENT: self.jarclient, SERVER: self.jarserver} kill_signatures(jars[side]) dir_bin = os.path.join(fml_dir, 'bin') if not os.path.isdir(dir_bin): os.makedirs(dir_bin) class_path = os.pathsep.join([f for f in self.cpathclient + [dir_bin] if not f in jars.values()]) dir_common = os.path.join(fml_dir, 'common') dir_trans = os.path.join(dir_common, 'cpw/mods/fml/common/asm/transformers'.replace('/', os.sep)) java = self.cmdjava.translate(None, '"') javac = self.cmdjavac.translate(None, '"') cmd_compile = '"%s" -Xlint:-options -deprecation -g -source 1.6 -target 1.6 -classpath "{classpath}" -sourcepath "{sourcepath}" -d "{outpath}" "{target}"' % javac cmd_compile = cmd_compile.format(classpath=class_path, sourcepath=dir_common, outpath=dir_bin, target="{target}") #Compile AccessTransformer and MCPMerger if we havent already # Only needs to happen once, but we don't know if were gunna decompile both client and server so cant do it based off side if compile_tools: self.logger.info('> Compiling AccessTransformer') if not runcmd(cmd_compile.format(target=os.path.join(dir_trans, 'AccessTransformer.java')), commands=self, echo=False): sys.exit(1) self.logger.info('> Compiling MCPMerger') if not runcmd(cmd_compile.format(target=os.path.join(dir_trans, 'MCPMerger.java')), commands=self, echo=False): sys.exit(1) compile_tools = False # Merge the client and server jar, only needs to be run once so only do it on the client if side == CLIENT: if not disable_merge: self.logger.info('> Running MCPMerger') forkcmd = ('"%s" -classpath "{classpath}" cpw.mods.fml.common.asm.transformers.MCPMerger "{mergecfg}" "{client}" "{server}"' % java).format( classpath=class_path, mergecfg=os.path.join(fml_dir, 'mcp_merge.cfg'), client=jars[CLIENT], server=jars[SERVER]) if not runcmd(forkcmd, echo=False, commands=self): sys.exit(1) else: self.logger.info('> MCPMerge disabled') apply_ats(fml_dir, mcp_dir, class_path, jars[side], disable_at=disable_at, commands=self) #Check the original jars not the transformed jars def checkjars_shunt(self, side, checkjars_real = Commands.checkjars): self.jarclient = self.jarclient + '.backup' self.jarserver = self.jarserver + '.backup' ret = checkjars_real(self, side) self.jarclient = self.jarclient[:-7] self.jarserver = self.jarserver[:-7] return ret try: pre_decompile(mcp_dir, fml_dir, disable_assets=disable_assets) os.chdir(mcp_dir) Commands.applyrg = applyrg_shunt Commands.applyss = applyss_shunt Commands.checkjars = checkjars_shunt #decompile -d -n -r # Conf JAD CSV -r -d -a -n -p -o -l -g -c -s --rg decompile(None, False, False, True, True, False, True, False, False, False, False, not disable_client, enable_server, False) reset_logger() os.chdir(fml_dir) post_decompile(mcp_dir, fml_dir) except SystemExit, e: print 'Decompile Exception: %d ' % e.code raise e if not os.path.isdir(src_dir): print 'Something went wrong, src folder not found at: %s' % src_dir sys.exit(1) #cleanup_source cleanup_source(src_dir) os.chdir(mcp_dir) commands = Commands(verify=True) if not disable_client: updatemd5_side(mcp_dir, commands, CLIENT) reset_logger() if enable_server: updatemd5_side(mcp_dir, commands, CLIENT) reset_logger() os.chdir(fml_dir) def updatemd5_side(mcp_dir, commands, side): sys.path.append(mcp_dir) from runtime.mcp import recompile_side, updatemd5_side from runtime.commands import SIDE_NAME recomp = recompile_side(commands, side) if recomp: commands.logger.info('> Generating %s md5s', SIDE_NAME[side]) commands.gathermd5s(side, skip_fml=True) def pre_decompile(mcp_dir, fml_dir, disable_assets=False): download_minecraft(mcp_dir, fml_dir) if not disable_assets: download_assets(mcp_dir) def post_decompile(mcp_dir, fml_dir): if False: print('hi') def apply_fml_patches(fml_dir, mcp_dir, src_dir, copy_files=True): #Delete /minecraft/cpw to get rid of the Side/SideOnly classes used in decompilation cpw_mc_dir = os.path.join(src_dir, 'minecraft', 'cpw') if os.path.isdir(cpw_mc_dir): shutil.rmtree(cpw_mc_dir) #patch files print('Applying Forge ModLoader patches') sys.stdout.flush() if os.path.isdir(os.path.join(fml_dir, 'patches', 'minecraft')): apply_patches(mcp_dir, os.path.join(fml_dir, 'patches', 'minecraft'), src_dir) if copy_files and os.path.isdir(os.path.join(fml_dir, 'client')): copytree(os.path.join(fml_dir, 'client'), os.path.join(src_dir, 'minecraft')) if copy_files and os.path.isdir(os.path.join(fml_dir, 'common')): copytree(os.path.join(fml_dir, 'common'), os.path.join(src_dir, 'minecraft')) #delete argo if os.path.isdir(os.path.join(src_dir, 'minecraft', 'argo')): shutil.rmtree(os.path.join(src_dir, 'minecraft', 'argo')) def finish_setup_fml(fml_dir, mcp_dir, enable_server=False, disable_client=False, disable_rename=False): sys.path.append(mcp_dir) from runtime.updatenames import updatenames from runtime.updatemd5 import updatemd5 from runtime.updatemcp import updatemcp os.chdir(mcp_dir) if not disable_rename: updatenames(None, True, not disable_client, enable_server) reset_logger() updatemd5(None, True, not disable_client, enable_server) reset_logger() os.chdir(fml_dir) #========================================================================== # Download Functions! #========================================================================== def download_minecraft(mcp_dir, fml_dir, version=None): mc_info = read_mc_versions(fml_dir, version=version, work_dir=os.path.join(mcp_dir, 'jars')) failed = False if mc_info['new_launcher']: if os.path.isdir(os.path.join(fml_dir, 'jsons')): json_file = os.path.join(fml_dir, 'jsons', '%s-dev.json' % mc_info['version']) else: json_file = os.path.join(fml_dir, 'fml.json') version_json = None try: version_json = json.load(open(json_file)) except Exception as e: print 'Failed to load version json: %s' % json_file sys.exit(1) failed = download_libraries(mcp_dir, version_json['libraries'], mc_info['natives_dir']) or failed if os.path.isfile(mc_info['json_file']): os.remove(mc_info['json_file']) shutil.copy(json_file, mc_info['json_file']) else: failed = not download_list(mc_info['downloads']) or failed # Remove any invalid files for type in ['client', 'server']: file_backup(mc_info['%s_file' % type], mc_info['%s_md5' % type]) failed = not download_file(mc_info['%s_url' % type], mc_info['%s_file' % type], mc_info['%s_md5' % type]) or failed file_backup(mc_info['%s_file' % type], mc_info['%s_md5' % type]) if failed: print 'Something failed verifying minecraft files, see log for details.' sys.exit(1) def download_libraries(mcp_dir, libraries, natives_dir): # Will attempt to download a list of maven style libraries from the default Minecraft website # or a custom website if the library specifies it # This list should be in the format of the new launcher's version.json file # Under the entry 'libraries' lib_dir = os.path.join(mcp_dir, 'jars', 'libraries') default_url = 'https://s3.amazonaws.com/Minecraft.Download/libraries' downloads = [] failed = False for lib in libraries: name = lib['name'].split(':') domain = name[0].split('.') root = name[1] version = name[2] path = domain + [root, version] extract = None root_url = default_url if 'extract' in lib.keys(): extract = lib['extract'] if 'url' in lib.keys(): root_url = lib['url'] file_names = ['%s-%s.jar' % (root, version)] if 'natives' in lib.keys(): file_names = [] for k,v in lib['natives'].items(): file_names.append('%s-%s-%s.jar' % (root, version, v)) if 'children' in lib.keys(): for child in lib['children']: file_names.append('%s-%s-%s.jar' % (root, version, child)) for file_name in file_names: url = '%s/%s/%s' % (root_url, '/'.join(path), file_name) file_path = os.path.join(lib_dir, os.sep.join(path), file_name) headers = get_headers(url) if headers is None: print('Could not retreive headers for library: %s ( %s )' % (lib['name'], url)) failed = True else: md5 = None if 'ETag' in headers.keys(): # Amazon headers, Mojang's server md5 = headers['ETag'] else: # Could be a normal maven repo, check for .md5 file try: md5 = urllib2.urlopen(url + '.md5').read().split(' ')[0].replace('\r', '').replace('\n', '') if not len(md5) == 32: md5 = None print('Could not retrieve md5 for library %s ( %s.md5 )' % (file_name, url)) failed = True except (HTTPError): failed = True pass downloads.append({ 'url' : url, 'file' : file_path, 'md5' : md5, 'size' : headers['Content-Length'], 'extract' : extract }) return download_list(downloads, natives_dir) or failed def download_list(list, natives_dir): #Downloads a list of files and urls. Verifying md5s if avalible. #Skipping already existing and valid files. #Also extracts files that are specified to be extracted to the natives folder missing = [] for dl in list: if os.path.isfile(dl['file']): if dl['md5'] is None or not get_md5(dl['file']) == dl['md5']: missing.append(dl) else: missing.append(dl) if len(missing) == 0: return False print 'Downloading %s libraries' % len(missing) failed = False for dl in missing: if download_file(dl['url'], dl['file'], dl['md5'], prefix=' '): if not dl['extract'] is None: excludes = [] if 'exclude' in dl['extract'].keys(): excludes = dl['extract']['exclude'] extract_zip(dl['file'], natives_dir, prefix=' ', filter=excludes) else: print(' Failed to download %s from %s' % (os.path.basename(dl['file']), dl['url'])) failed = True return failed def download_assets(mcp_dir): from xml.dom.minidom import parse asset_dir = os.path.join(mcp_dir, 'jars', 'assets') base_url = 'https://s3.amazonaws.com/Minecraft.Resources' print('Gathering assets list from %s' % base_url) files = [] failed = False try: url = urllib.urlopen(base_url) xml = parse(url) def get(xml, key): return xml.getElementsByTagName(key)[0].firstChild.nodeValue for asset in xml.getElementsByTagName('Contents'): path = get(asset, 'Key') if path.endswith('/'): continue file = os.path.join(asset_dir, os.sep.join(path.split('/'))) md5 = get(asset, 'ETag').replace('"', '') if os.path.isfile(file): if get_md5(file) == md5: continue files.append({ 'file' : file, 'url' : '%s/%s' % (base_url, path), 'size' : get(asset, 'Size'), 'md5' : md5 }) except Exception as e: print 'Error gathering asset list:' pprint(e) sys.exit(1) if len(files) == 0: print(' No new assets need to download') return print(' Downloading %s assets' % len(files)) for file in files: failed = not download_file(file['url'], file['file'], file['md5'], root=asset_dir, prefix=' ') or failed if failed: print(' Downloading assets failed, please review log for more details') sys.exit(1) #========================================================================== # Transformers #========================================================================== def apply_ats(fml_dir, mcp_dir, class_path, target, disable_at=False, commands=None): def log(msg): if commands is None: print(msg) else: commands.logger.info(msg) cmds = find_java() if cmds is None: log('>Could not run Access Transformer, Java not found!') sys.exit(1) if not disable_at: log('> Running AccessTransformer') forkcmd = ('"%s" -classpath "{classpath}" cpw.mods.fml.common.asm.transformers.AccessTransformer "{jar}" "{fmlconfig}"' % cmds['java']).format( classpath=class_path, jar=target, fmlconfig=os.path.join(fml_dir, 'common', 'fml_at.cfg')) forge_cfg = os.path.join(fml_dir, '..', 'common', 'forge_at.cfg') if os.path.isfile(forge_cfg): log(' Forge config detected') forkcmd += ' "%s"' % forge_cfg for dirname, dirnames, filenames in os.walk(os.path.join(fml_dir, '..', 'accesstransformers')): for filename in filenames: accesstransformer = os.path.join(dirname, filename) if os.path.isfile(accesstransformer): log(' Access Transformer "%s" detected' % filename) forkcmd += ' "%s"' % accesstransformer if not runcmd(forkcmd, echo=False, commands=commands): sys.exit(1) else: log('> Access Transformer disabled')