Inital script side commit for MC source repackaging done.

This commit is contained in:
LexManos 2012-11-20 01:55:41 -08:00
parent 397d36e900
commit 73a22bb571
6 changed files with 2872 additions and 169 deletions

View file

@ -9,26 +9,7 @@
if not sys.platform.startswith('win'):
if os.path.exists(path):
shutil.rmtree(path)
@@ -584,6 +586,9 @@
self.has_astyle_cfg = False
if os.path.isfile(self.astyleconf):
self.has_astyle_cfg = True
+
+ #FML Entries
+ self.srcshared = os.path.normpath(os.path.join(self.dirsrc, 'common'))
def creatergcfg(self, reobf=False, keep_lvt=False, keep_generics=False, rg_update=False):
"""Create the files necessary for RetroGuard"""
@@ -781,13 +786,16 @@
testlk = {CLIENT: self.testclient, SERVER: self.testserver}
if not os.path.exists(os.path.join(srclk[side], os.path.normpath(testlk[side] + '.java'))):
- return False
+ if not os.path.exists(os.path.join(self.srcshared, os.path.normpath(testlk[side] + '.java'))):
+ return False
return True
def checkbins(self, side):
@@ -788,6 +790,8 @@
binlk = {CLIENT: self.binclient, SERVER: self.binserver}
testlk = {CLIENT: self.testclient, SERVER: self.testserver}
@ -37,7 +18,7 @@
if not os.path.exists(os.path.join(binlk[side], os.path.normpath(testlk[side] + '.class'))):
return False
return True
@@ -1029,6 +1037,10 @@
@@ -1029,6 +1033,10 @@
pathsrclk = {CLIENT: self.srcclient, SERVER: self.srcserver}
pathlog = {CLIENT: self.clientrecomplog, SERVER: self.serverrecomplog}
@ -48,33 +29,16 @@
if not os.path.exists(pathbinlk[side]):
os.makedirs(pathbinlk[side])
@@ -1041,6 +1053,7 @@
all_files = True
append_pattern = False
pkglist = filterdirs(pathsrclk[side], '*.java', append_pattern=append_pattern, all_files=all_files)
+ pkglist = pkglist + filterdirs(self.srcshared, '*.java', append_pattern=append_pattern, all_files=all_files) #FML, Add Common folder
dirs = ' '.join(pkglist)
classpath = os.pathsep.join(cplk[side])
forkcmd = self.cmdrecomp.format(classpath=classpath, sourcepath=pathsrclk[side], outpath=pathbinlk[side],
@@ -1062,7 +1075,7 @@
@@ -1062,7 +1070,7 @@
raise
def startserver(self):
- classpath = [self.binserver] + self.cpathserver
+ classpath = [self.binclient, self.srcshared] + self.cpathserver
+ classpath = [self.binclient] + self.cpathserver
classpath = [os.path.join('..', p) for p in classpath]
classpath = os.pathsep.join(classpath)
os.chdir(self.dirjars)
@@ -1070,7 +1083,7 @@
self.runmc(forkcmd)
def startclient(self):
- classpath = [self.binclient] + self.cpathclient
+ classpath = [self.binclient, self.srcshared] + self.cpathclient
classpath = [os.path.join('..', p) for p in classpath]
classpath = os.pathsep.join(classpath)
natives = os.path.join('..', self.dirnatives)
@@ -1197,20 +1210,20 @@
@@ -1197,20 +1205,20 @@
with open(self.csvmethods, 'rb') as fh:
methodsreader = csv.DictReader(fh)
for row in methodsreader:
@ -98,56 +62,7 @@
names['params'][row['param']] = row['name']
regexps = {
@@ -1240,6 +1253,11 @@
# HINT: We pathwalk the sources
for path, _, filelist in os.walk(pathsrclk[side], followlinks=True):
+ for cur_file in fnmatch.filter(filelist, '*.java'):
+ updatefile(os.path.normpath(os.path.join(path, cur_file)))
+
+ # FML, copy of the above, for the common folder
+ for path, _, filelist in os.walk(self.srcshared, followlinks=True):
for cur_file in fnmatch.filter(filelist, '*.java'):
updatefile(os.path.normpath(os.path.join(path, cur_file)))
return True
@@ -1322,12 +1340,14 @@
pathsrclk = {CLIENT: self.srcclient, SERVER: self.srcserver}
strip_comments(pathsrclk[side])
+ strip_comments(self.srcshared)
def process_cleanup(self, side):
"""Do lots of random cleanups including stripping comments, trailing whitespace and extra blank lines"""
pathsrclk = {CLIENT: self.srcclient, SERVER: self.srcserver}
src_cleanup(pathsrclk[side], fix_imports=True, fix_unicode=True, fix_charval=True, fix_pi=True, fix_round=False)
+ src_cleanup(self.srcshared, fix_imports=True, fix_unicode=True, fix_charval=True, fix_pi=True, fix_round=False)
def process_javadoc(self, side):
"""Add CSV descriptions to methods and fields as javadoc"""
@@ -1336,6 +1356,21 @@
if not self.has_doc_csv:
self.logger.warning('!! javadoc disabled due to no csvs !!')
return False
+
+ #FML Recall this function on the common folder
+ #Potential bug: If this is called without a subsiquent rename call, will cause double comments
+ if pathsrclk[side] != self.srcshared:
+ if side == CLIENT:
+ tmp = self.srcclient
+ self.srcclient = self.srcshared
+ self.process_javadoc(side)
+ self.srcclient = tmp
+ else:
+ tmp = self.srcserver
+ self.srcserver = self.srcshared
+ self.process_javadoc(side)
+ self.srcserver = tmp
+
#HINT: We read the relevant CSVs
methodsreader = csv.DictReader(open(self.csvmethods, 'r'))
@@ -1344,13 +1379,13 @@
@@ -1344,13 +1352,13 @@
methods = {}
for row in methodsreader:
#HINT: Only include methods that have a non-empty description
@ -163,13 +78,7 @@
fields[row['searge']] = row['desc'].replace('*/', '* /')
regexps = {
@@ -1422,12 +1457,13 @@
# HINT: We create the list of source directories based on the list of packages
pkglist = filterdirs(pathsrclk[side], '*.java', append_pattern=True)
+ pkglist = pkglist + filterdirs(self.srcshared, '*.java', append_pattern=True) #FML, Add Common folder
dirs = ' '.join(pkglist)
forkcmd = self.cmdastyle.format(classes=dirs, conffile=self.astyleconf)
@@ -1427,7 +1435,7 @@
self.runcmd(forkcmd)
return True
@ -178,7 +87,7 @@
if not reobf:
md5lk = {CLIENT: self.md5client, SERVER: self.md5server}
else:
@@ -1442,6 +1478,9 @@
@@ -1442,6 +1450,9 @@
class_path = ''
else:
class_path += '/'
@ -188,7 +97,7 @@
for class_file in fnmatch.filter(filelist, '*.class'):
class_name = class_path + os.path.splitext(class_file)[0]
bin_file = os.path.normpath(os.path.join(path, class_file))
@@ -1594,6 +1633,9 @@
@@ -1594,6 +1605,9 @@
sys.exit(1)
for entry in newfiles:
@ -198,7 +107,7 @@
if entry[3] == 'U':
self.logger.info('Retrieving file from server : %s', entry[0])
cur_file = os.path.normpath(entry[0])
@@ -1614,6 +1656,9 @@
@@ -1614,6 +1628,9 @@
md5reoblk = {CLIENT: self.md5reobfclient, SERVER: self.md5reobfserver}
outpathlk = {CLIENT: self.srcmodclient, SERVER: self.srcmodserver}
src = {CLIENT: self.srcclient, SERVER: self.srcserver}
@ -208,28 +117,10 @@
# HINT: We need a table for the old md5 and the new ones
md5table = {}
@@ -1646,6 +1691,7 @@
# HINT: We extract the source files for the modified class files
for in_class in trgclasses:
src_file = os.path.normpath(os.path.join(src[side], in_class + '.java'))
+ cmn_file = os.path.normpath(os.path.join(self.srcshared, in_class + '.java'))
dest_file = os.path.normpath(os.path.join(outpathlk[side], in_class + '.java'))
if os.path.isfile(src_file):
if not os.path.exists(os.path.dirname(dest_file)):
@@ -1655,4 +1701,15 @@
self.logger.info('> Outputted %s to %s', in_class.ljust(35), outpathlk[side])
@@ -1656,3 +1673,6 @@
except IOError:
self.logger.error('* File %s copy failed', in_class)
-
+ elif os.path.isfile(cmn_file):
+ if not os.path.exists(os.path.dirname(dest_file)):
+ os.makedirs(os.path.dirname(dest_file))
+ try:
+ shutil.copyfile(cmn_file, dest_file)
+ self.logger.info('> Outputted %s to %s', in_class.ljust(35), outpathlk[side])
+ except IOError:
+ self.logger.error('* File %s copy failed', in_class)
+
+def commands_sanity_check():
+ print 'Commands patch applied successfully'
+

View file

@ -22,8 +22,8 @@ Params = %(DirConf)s/params.csv
NewIds = %(DirConf)s/newids.csv
[SRGS]
ConfClient = %(DirConf)s/joined.srg
ConfServer = %(DirConf)s/joined.srg
ConfClient = %(DirConf)s/packaged.srg
ConfServer = %(DirConf)s/packaged.srg
Client = %(DirTemp)s/client_rg.srg
Server = %(DirTemp)s/server_rg.srg
DeobfClient = %(DirTemp)s/client_deobf.srg
@ -56,8 +56,8 @@ ServerDeobLog = %(DirLogs)s/server_deob.log
NullPkg = net/minecraft/src
[EXCEPTOR]
XClientCfg = %(DirConf)s/joined.exc
XServerCfg = %(DirConf)s/joined.exc
XClientCfg = %(DirConf)s/packaged.exc
XServerCfg = %(DirConf)s/packaged.exc
XClientOut = %(DirTemp)s/minecraft_exc.jar
XServerOut = %(DirTemp)s/minecraft_server_exc.jar
XClientLog = %(DirLogs)s/client_exc.log

1327
fml/conf/packages.csv Normal file

File diff suppressed because it is too large Load diff

View file

@ -329,8 +329,6 @@ def setup_fml(fml_dir, mcp_dir):
#cleanup_source
cleanup_source(src_dir)
merge_client_server(mcp_dir)
os.chdir(mcp_dir)
commands = Commands(verify=True)
updatemd5_side(mcp_dir, commands, CLIENT)
@ -386,48 +384,6 @@ def get_joined_srg(mcp_dir):
values[pts[0]][pts[1]] = pts[2]
return values
def merge_client_server(mcp_dir):
client = os.path.join(mcp_dir, 'src', 'minecraft')
shared = os.path.join(mcp_dir, 'src', 'common')
client_jar = os.path.join(mcp_dir, 'jars', 'bin', 'minecraft.jar')
server_jar = os.path.join(mcp_dir, 'jars', 'minecraft_server.jar')
joined_srg = get_joined_srg(mcp_dir)['CL:']
if not os.path.isfile(client_jar) or not os.path.isfile(server_jar):
return
if not os.path.isdir(shared):
os.makedirs(shared)
server_classes = []
zip = ZipFile(server_jar)
for i in zip.filelist:
if i.filename.endswith('.class'):
server_classes.append(i.filename[:-6])
for cls in server_classes:
if cls in joined_srg.keys():
cls = joined_srg[cls]
cls += '.java'
f_client = os.path.normpath(os.path.join(client, cls.replace('/', os.path.sep))).replace(os.path.sep, '/')
f_shared = os.path.normpath(os.path.join(shared, cls.replace('/', os.path.sep))).replace(os.path.sep, '/')
if not os.path.isfile(f_client):
print 'Issue Merging File Not Found: ' + cls
continue
if not cls.rfind('/') == -1:
new_dir = os.path.join(shared, cls.rsplit('/', 1)[0])
if not os.path.isdir(new_dir):
os.makedirs(new_dir)
shutil.move(f_client, f_shared)
cleanDirs(client)
def apply_fml_patches(fml_dir, mcp_dir, src_dir, copy_files=True):
#Delete /common/cpw to get rid of the Side/SideOnly classes used in decompilation
@ -687,6 +643,8 @@ def setup_mcp(fml_dir, mcp_dir, dont_gen_conf=True):
print 'Copying FML conf'
shutil.copytree(fml_conf, mcp_conf)
gen_renamed_conf(mcp_dir, fml_dir)
#update workspace
print 'Fixing MCP Workspace'
if not os.path.isdir(os.path.join(fml_dir, 'eclipse', 'Clean-Client')):
@ -838,3 +796,106 @@ def gen_merged_csv(common_map, in_file, out_file, main_key='searge'):
writer.writeheader()
for row in sorted(common, key=lambda row: row[main_key]):
writer.writerow(row)
def repackage_class(pkgs, cls):
if cls.startswith('net/minecraft/src/'):
tmp = cls[18:]
if tmp in pkgs.keys():
return '%s/%s' % (pkgs[tmp], tmp)
return cls
typere = re.compile('([\[ZBCSIJFDV]|L([\w\/]+);)')
def repackage_signature(pkgs, sig):
global typere
sig1 = sig
params = sig.rsplit(')', 1)[0][1:]
ret = sig.rsplit(')', 1)[1]
sig = '('
for arg in typere.findall(params):
if len(arg[1]) > 0:
sig += 'L%s;' % repackage_class(pkgs, arg[1])
else:
sig += arg[0]
sig += ')'
for tmp in typere.findall(ret):
if len(tmp[1]) > 0:
sig += 'L%s;' % repackage_class(pkgs, tmp[1])
else:
sig += tmp[0]
return sig
def gen_renamed_conf(mcp_dir, fml_dir):
pkg_file = os.path.join(fml_dir, 'conf', 'packages.csv')
srg_in = os.path.join(mcp_dir, 'conf', 'joined.srg')
srg_out = os.path.join(mcp_dir, 'conf', 'packaged.srg')
exc_in = os.path.join(mcp_dir, 'conf', 'joined.exc')
exc_out = os.path.join(mcp_dir, 'conf', 'packaged.exc')
pkgs = {}
if os.path.isfile(pkg_file):
with open(pkg_file) as fh:
reader = csv.DictReader(fh)
for line in reader:
pkgs[line['class']] = line['package']
print 'Creating re-packaged srg'
with open(srg_in, 'r') as inf:
with open(srg_out, 'wb') as outf:
for line in inf:
pts = line.rstrip('\r\n').split(' ')
if pts[0] == 'PK:':
outf.write(' '.join(pts) + '\n')
elif pts[0] == 'CL:':
pts[2] = repackage_class(pkgs, pts[2])
outf.write('CL: %s %s\n' % (pts[1], pts[2]))
elif pts[0] == 'FD:':
tmp = pts[2].rsplit('/', 1)
tmp[0] = repackage_class(pkgs, tmp[0])
outf.write('FD: %s %s/%s\n' % (pts[1], tmp[0], tmp[1]))
elif pts[0] == 'MD:':
tmp = pts[3].rsplit('/', 1)
pts[3] = '%s/%s' % (repackage_class(pkgs, tmp[0]), tmp[1])
pts[4] = repackage_signature(pkgs, pts[4])
outf.write('MD: %s %s %s %s\n' % (pts[1], pts[2], pts[3], pts[4]))
else:
print 'Line unknown in SRG: ' + line
outf.write(line)
excre = re.compile('([\[ZBCSIJFDV]|L([\w\/]+);)')
print 'Creating re-packaged exc'
with open(exc_in, 'r') as inf:
with open(exc_out, 'wb') as outf:
for line in inf:
line = line.rstrip('\r\n')
cls = line.split('.')[0]
named = line.rsplit('=', 1)[1]
line = line[len(cls)+1:-1*len(named)-1]
func = line.split('(')[0]
tmp = named.split('|', 1)
if len(tmp[0]) > 0:
excs = tmp[0].split(',')
for x in range(len(excs)):
excs[x] = repackage_class(pkgs, excs[x])
named = '%s|%s' % (','.join(excs), tmp[1])
sig = repackage_signature(pkgs, line[len(func):])
cls = repackage_class(pkgs, cls)
outf.write('%s.%s%s=%s\n' % (cls, func, sig, named))
print 'Creating re-packaged MCP patch'
patch_in = os.path.join(mcp_dir, 'conf', 'patches', 'minecraft_ff.patch')
patch_tmp = os.path.join(mcp_dir, 'conf', 'patches', 'minecraft_ff.patch.tmp')
regnms = re.compile(r'net\\minecraft\\src\\(\w+)')
with open(patch_in, 'r') as fh:
buf = fh.read()
def mapname(match):
return repackage_class(pkgs, match.group(0).replace('\\', '/')).replace('/', '\\')
buf = regnms.sub(mapname, buf)
with open(patch_tmp, 'w') as fh:
fh.write(buf)
shutil.move(patch_tmp, patch_in)

1327
fml/packages.cfg Normal file

File diff suppressed because it is too large Load diff

97
fml/update_packages.py Normal file
View file

@ -0,0 +1,97 @@
import os, os.path, sys
import shutil, glob, fnmatch
import csv, re
from pprint import pprint
from zipfile import ZipFile
def get_merged_info():
mcp_dir = os.path.join(os.getcwd(), '..')
joined = os.path.join(mcp_dir, 'conf', 'joined.srg')
values = {'PK:': {}, 'CL:': {}, 'FD:': {}, 'MD:': {}}
if not os.path.isfile(joined):
sys.path.append('install')
from fml import gen_merged_srg
values = gen_merged_srg(mcp_dir, None)
else:
with open(joined, 'r') as fh:
for line in fh:
pts = line.rstrip('\r\n').split(' ')
if pts[0] == 'MD:':
values[pts[0]][pts[1] + ' ' + pts[2]] = pts[3] + ' ' + pts[4]
else:
values[pts[0]][pts[1]] = pts[2]
return {t:{v.split(' ')[0]:k for k, v in m.items()} for t,m in values.items()}
def main():
client_jar = os.path.join('..', 'jars', 'bin', 'minecraft.jar.backup')
server_jar = os.path.join('..', 'jars', 'minecraft_server.jar.backup')
server_classes = []
client_classes = []
zip = ZipFile(client_jar)
for i in zip.filelist:
if i.filename.endswith('.class') and i.filename.find('/') == -1:
client_classes.append(i.filename[:-6])
zip.close()
zip = ZipFile(server_jar)
for i in zip.filelist:
if i.filename.endswith('.class') and i.filename.find('/') == -1:
server_classes.append(i.filename[:-6])
zip.close()
srg = get_merged_info()
pkgs = {}
pkg_file = os.path.join('conf', 'packages.csv')
if os.path.isfile(pkg_file):
with open(pkg_file) as fh:
reader = csv.DictReader(fh)
for line in reader:
pkgs[line['class']] = line['package']
classes = []
for cls in srg['CL:'].keys():
if cls.startswith('net/minecraft/src/'):
obf = srg['CL:'][cls]
cls = cls[18:]
classes.append(cls)
if obf in server_classes:
server_classes.remove(obf)
server_classes.append(cls)
if obf in client_classes:
client_classes.remove(obf)
client_classes.append(cls)
for cls in pkgs.keys():
if not cls in classes:
print 'Removed Class: %s/%s' % (key, pkgs[key])
for cls in classes:
if not cls in pkgs.keys():
print 'New Class: %s' % cls
if cls.find('/') == -1:
if cls in server_classes and cls in client_classes:
pkgs[cls] = 'net/minecraft/shared'
elif cls in server_classes:
pkgs[cls] = 'net/minecraft/server'
elif cls in client_classes:
pkgs[cls] = 'net/minecraft/client'
else:
pkgs[cls] = 'net/minecraft/src'
tmp=[]
for cls,pkg in pkgs.items():
tmp.append({'class': cls, 'package': pkg})
with open(pkg_file, 'wb') as fh:
writer = csv.DictWriter(fh, fieldnames=['class', 'package'], lineterminator='\n')
writer.writeheader()
for row in sorted(tmp, key=lambda x: (x['package'], x['class'])):
writer.writerow(row)
if __name__ == '__main__':
main()