anonymous group photoblog software
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

584 lines
19 KiB

  1. #!/usr/bin/env python
  2. import os
  3. import re
  4. import optparse
  5. try:
  6. from cStringIO import StringIO
  7. except ImportError:
  8. from StringIO import StringIO
  9. try:
  10. import config, config_defaults
  11. except ImportError:
  12. config = None
  13. TEMPLATES_DIR = 'templates'
  14. HTDOCS_HARDCODED_PATH = '/home/desuchan/public_html/desuchan.net/htdocs/'
  15. FUTABA_STYLE_DEBUG = 0
  16. EXPRESSION_DEBUG = 0
  17. EXPRESSION_TRANSLATOR_DEBUG = 0
  18. LOOP_TAG_DEBUG = 0
  19. VARIABLES_DEBUG = 0
  20. TEMPLATE_RE = re.compile(r'^use constant ([A-Z_]+) => (.*?;)\s*\n\n', re.M | re.S)
  21. TEMPLATE_SECTION_RE = re.compile(
  22. r'('
  23. r'q{((?:[^{}]|{[^{}]*})*)}|' # allow non-nested braces inside the q{}
  24. r'include\("([a-z_/\.]*?)"\)|'
  25. r'"(.*?)"|'
  26. r'([A-Z][A-Z_]+)|'
  27. r'sprintf\(S_ABBRTEXT,([\'"].*?[\'"])\)'
  28. r')[\.;] *',
  29. re.S | re.M)
  30. COMPILE_TEMPLATE_RE = re.compile(
  31. r'^compile_template ?\((.*?)\);$',
  32. re.S)
  33. # regex copied from wakautils.pl
  34. TAG_RE = re.compile(
  35. r'(.*?)(<(/?)(var|const|if|loop)(?:|\s+(.*?[^\\]))>|$)',
  36. re.S)
  37. # i should write a decent tokenizer/parser instead
  38. PERL_EXP_RE = re.compile(
  39. # 1-2 board option, path
  40. r'\$board->option\((\'[A-Z_]+\'|"[A-Z_]+")\)|'
  41. r'\$board->(path\(?\)?)|'
  42. # 3 advanced include (ignore most of it)
  43. r'encode_string\(\(compile_template\(include\(\$board->path\(\).\'/\'.'
  44. '"([a-z/\.]+?)"\)\)\)->\(board=>\$board\)\)|'
  45. # 4-5 function call (evaluate recursively)
  46. r'([a-z_]+)\(|'
  47. r'(\))|'
  48. # 6-7 variables and constants
  49. r'\$([A-Za-z0-9_{}]+)|'
  50. r'([A-Z_]+)|'
  51. # 8 sprintf without brackets
  52. r'sprintf (.+)$|'
  53. # 9 regex
  54. r'([!=]~ /.+?/[i]*)|'
  55. # 10-11 operators and comma
  56. r'(\+|-|/|\*|<=|>=|<|>|==|eq|!=|ne|&&|and|\|\||or|!|\?|:|\.)|'
  57. r'(,)|'
  58. # 12-13 values (string/number), whitespace
  59. r'(".*?"|\'.*?\'|[0-9]+)|'
  60. r'(\s+)|'
  61. # 14 single opening bracket (turns into void function)
  62. r'(\()',
  63. re.S | re.M)
  64. # post and admin table columns
  65. _POST_TABLE = ['num', 'parent', 'timestamp', 'lasthit', 'ip', 'date', 'name',
  66. 'trip', 'email', 'subject', 'password', 'comment', 'image', 'size', 'md5',
  67. 'width', 'height', 'thumbnail', 'tn_width', 'tn_height', 'lastedit',
  68. 'lastedit_ip', 'admin_post', 'stickied', 'locked']
  69. _ADMIN_TABLE = ['username', 'num', 'type', 'comment', 'ival1', 'ival2',
  70. 'sval1', 'total', 'expiration', 'divider']
  71. # oh god what is this
  72. KNOWN_LOOPS = {
  73. 'stylesheets': ('stylesheet', ['filename', 'title', 'default']),
  74. 'inputs': ('input', ['name', 'value']),
  75. 'S_OEKPAINTERS': ('painters', ['painter', 'name']),
  76. 'threads': ('currentthread', ['posts', 'omit', 'omitimages']),
  77. 'posts': ('post', _POST_TABLE + ['abbrev', 'postnum']),
  78. 'pages': ('page', ['page', 'filename', 'current']),
  79. 'loop': ('post', _POST_TABLE),
  80. 'boards_select': ('board', ['board_entry']),
  81. 'reportedposts': ('rpost', ['reporter', 'offender', 'postnum', 'comment',
  82. 'date', 'resolved']),
  83. 'users': ('user', ['username', 'account', 'password', 'reign', 'disabled']),
  84. 'boards': ('board', ['board_entry', 'underpower']),
  85. 'staff': ('account', ['username']),
  86. # this is actually three different loops
  87. 'entries': ('entry', ['num', 'username', 'action', 'info', 'date', 'ip',
  88. 'admin_id', 'timestamp', 'rowtype', 'disabled',
  89. 'account', 'expiration', 'id', 'host', 'task',
  90. 'boardname', 'post', 'timestamp', 'passfail']),
  91. 'edits': ('edit', ['username', 'date', 'info', 'num']),
  92. 'bans': ('ban', _ADMIN_TABLE + ['rowtype', 'expirehuman', 'browsingban']),
  93. 'hash': ('row', _ADMIN_TABLE),
  94. 'scanned': ('proxy', ['num', 'type', 'ip', 'timestamp',
  95. 'date', 'divider', 'rowtype']),
  96. 'errors': ('error', ['error']),
  97. 'items': ('post', _POST_TABLE + ['mime_type']),
  98. 'reports' : ('report', ['reporter', 'offender', 'postnum', 'comment',
  99. 'date', 'resolved', 'board_name'])
  100. }
  101. RENAME = {
  102. 'sprintf': 'format',
  103. 'OEKAKI_DEFAULT_PAINTER': 'board.options.OEKAKI_DEFAULT_PAINTER',
  104. 'ENV{SERVER_NAME}': "environ['SERVER_NAME']",
  105. 'ENV{HTTP_REFERER}': "environ['HTTP_REFERER']",
  106. 'self': "get_script_name()",
  107. 'escamp': 'escape',
  108. 'expand_filename': 'expand_url',
  109. 'expand_image_filename': 'expand_image_url',
  110. 'round_decimal': 'round',
  111. 'get_filename': 'basename',
  112. 'include/boards/announcements_global.html': 'announcements_global.html',
  113. 'include/announcements.html': 'announcements.html',
  114. '../include/boards/rules.html': 'rules.html',
  115. }
  116. TOUCHUPS = {
  117. # Fix savelogin checkbox in admin login.
  118. '"savelogin"': '"savelogin" value="1"',
  119. # wakaba.pl -> wakarimasen.py
  120. 'wakaba.pl': '{{ get_script_name() }}',
  121. # Replace references to wakaba with wakarimasen.
  122. 'wakaba': 'wakarimasen',
  123. # Extend post.comment expression with tag filter for abbreviated pages.
  124. '{{ post.comment }}': '{% if omit %}'
  125. '{{ post.comment|redirect_reply_links(min_res) }}'
  126. '{% else %}{{ post.comment }}{% endif %}',
  127. # Fix for abbreviated thread message.
  128. 'if thread and currentthread.omit': 'if thread and omit',
  129. 'For the other {{ currentthread.omit }}': 'For the other {{ omit }}',
  130. }
  131. REMOVE_BACKSLASHES_RE = re.compile(r'\\([^\\])')
  132. def remove_backslashes(string):
  133. return REMOVE_BACKSLASHES_RE.sub(r'\1', string)
  134. def debug_item(name, value='', match=None, span=''):
  135. span = match and match.span() or span
  136. if value:
  137. value = repr(value)[1:-1]
  138. if len(value) > 50:
  139. value = value[:50] + "[...]"
  140. print ' %14s %-8s %s' % (span, name, value)
  141. class FutabaStyleParser(object):
  142. def __init__(self, filename="futaba_style.pl", only=None, dry_run=False):
  143. self.only = only
  144. self.dry_run = dry_run
  145. self.lastend = 0
  146. self.current = None
  147. if not os.path.exists(TEMPLATES_DIR) and not self.dry_run:
  148. os.mkdir(TEMPLATES_DIR)
  149. self.tl = Jinja2Translator(self)
  150. TEMPLATE_RE.sub(self.do_constant, open(filename).read())
  151. def debug_item(self, *args, **kwds):
  152. if not FUTABA_STYLE_DEBUG:
  153. return
  154. debug_item(*args, **kwds)
  155. def do_constant(self, match):
  156. name, template = match.groups()
  157. if self.only and self.only != name:
  158. return
  159. if FUTABA_STYLE_DEBUG or LOOP_TAG_DEBUG or VARIABLES_DEBUG:
  160. print name
  161. # remove compile_template(...)
  162. compile = COMPILE_TEMPLATE_RE.match(template)
  163. if compile:
  164. self.debug_item('compiled', '1')
  165. template = compile.group(1) + ';'
  166. # init variables for the self.do_section loop
  167. self.lastend = 0
  168. self.current = StringIO()
  169. TEMPLATE_SECTION_RE.sub(self.do_section, template)
  170. # after the self.do_section loop
  171. current = self.current.getvalue()
  172. current = self.parse_template_tags(current)
  173. current = self.do_touchups(current)
  174. if not self.dry_run:
  175. file = open(template_filename(name), 'w')
  176. file.write(current)
  177. if len(template) != self.lastend:
  178. self.debug_item("NOT MATCHED (end)", template[self.lastend:],
  179. span=(self.lastend, len(template)))
  180. def do_section(self, match):
  181. if not match.group():
  182. return
  183. if match.start() > self.lastend:
  184. span = (self.lastend, match.start())
  185. self.debug_item("NOT MATCHED", match.string[span[0]:span[1]],
  186. span=span)
  187. names = ['html', 'include', 'string', 'const', 'abbrtext']
  188. groups = list(match.groups())[1:]
  189. for groupname, value in map(None, names, groups):
  190. if value:
  191. self.debug_item(groupname, value, match)
  192. self.current.write(self.tl.handle_item(groupname, value))
  193. self.lastend = match.end()
  194. def parse_template_tags(self, template):
  195. return TemplateTagsParser(self.tl).run(template)
  196. def do_touchups(self, template):
  197. for old, new in TOUCHUPS.items():
  198. template = template.replace(old, new)
  199. return template
  200. class TemplateTagsParser(object):
  201. def __init__(self, tl):
  202. self.tl = tl
  203. self.output = None
  204. self.loops = []
  205. def run(self, template):
  206. self.output = StringIO()
  207. for match in TAG_RE.finditer(template):
  208. html, tag, closing, name, args = match.groups()
  209. if html:
  210. self.output.write(html)
  211. if args:
  212. args = remove_backslashes(args)
  213. if tag:
  214. if closing:
  215. self.end_tag(name)
  216. else:
  217. self.start_tag(tag, name, args)
  218. return self.output.getvalue()
  219. def start_tag(self, tag, name, args):
  220. template = self.tl.TAGS[name][0]
  221. try:
  222. args = self.tl.translate_expression(self.parse_expression(args),
  223. name, self.loops)
  224. except AdvInclude, e:
  225. template = self.tl.TAGS['include']
  226. args = self.tl.handle_include(e.value)
  227. if name == 'loop':
  228. if LOOP_TAG_DEBUG:
  229. print "Enter loop", args
  230. self.loops.append(args[1].split('.')[-1])
  231. self.output.write(template % args)
  232. def end_tag(self, name):
  233. if name == 'loop':
  234. loop = self.loops.pop()
  235. if LOOP_TAG_DEBUG:
  236. print "Exit loop", loop
  237. self.output.write(self.tl.TAGS[name][1])
  238. def parse_expression(self, exp):
  239. lastend = 0
  240. if EXPRESSION_DEBUG or EXPRESSION_TRANSLATOR_DEBUG:
  241. print "Expression\t", exp
  242. result = self.parse_subexpression(exp)[0]
  243. if EXPRESSION_DEBUG:
  244. print ' ', result
  245. return result
  246. def parse_subexpression(self, exp, tmp=None):
  247. '''return value: tuple
  248. [0] list of tokens
  249. [1] the remaining
  250. if tmp is set, results are appended to that list instead of returning
  251. a new one (useful when parsing the remaining)
  252. '''
  253. lastend = 0
  254. if tmp is None:
  255. result = []
  256. else:
  257. result = tmp
  258. for match in PERL_EXP_RE.finditer(exp):
  259. if not match.group():
  260. continue
  261. if EXPRESSION_DEBUG and match.start() > lastend:
  262. span = (lastend, match.start())
  263. debug_item("unknown token", match.string[span[0]:span[1]],
  264. span=span)
  265. names = ['option', 'path', 'advinclude', 'function', 'funcend',
  266. 'var', 'const', 'sprintf', 'regex', 'operator', 'comma',
  267. 'value', 'whitespace', 'void']
  268. groups = match.groups()
  269. for groupname, value in map(None, names, groups):
  270. if value:
  271. break
  272. retval = self.handle_token(groupname, value, match, result)
  273. if retval is not None:
  274. return retval
  275. lastend = match.end()
  276. if EXPRESSION_DEBUG and len(exp) != lastend:
  277. debug_item("unknown token", exp[lastend:],
  278. span=(lastend, len(exp)))
  279. return (result, '')
  280. def call_function(self, name, args, result):
  281. function, remaining = self.parse_subexpression(args)
  282. result.append(('function', (name, function)))
  283. return self.parse_subexpression(remaining, result)
  284. def handle_token(self, type, value, match, result):
  285. if type == 'sprintf':
  286. return self.call_function('sprintf', value + ')', result)
  287. elif type == 'void':
  288. type, value = 'function', 'void'
  289. if type == 'function':
  290. return self.call_function(value, match.string[match.end():],
  291. result)
  292. elif type == 'funcend':
  293. remaining = match.string[match.end():]
  294. return (result, remaining)
  295. if type == 'option':
  296. value = value.strip('\'"')
  297. if type == 'regex':
  298. if value.startswith("!"):
  299. result.append(('operator', '!'))
  300. value = value[2:].strip(' ')
  301. if type != 'whitespace':
  302. result.append((type, value))
  303. class Jinja2Translator(object):
  304. '''Just to keep jinja2-specific code separate'''
  305. TAGS = {
  306. 'var': ('{{ %s }}', ''),
  307. 'const': ('{{ %s }}', ''),
  308. 'if': ('{%% if %s %%}', '{% endif %}'),
  309. 'loop': ('{%% for %s in %s %%}', '{% endfor %}'),
  310. 'include': "{%% include '%s' %%}",
  311. 'filter': '{%% filter %s %%}%s{%% endfilter %%}',
  312. }
  313. OPERATORS = {
  314. '!': 'not',
  315. 'eq': '==',
  316. 'ne': '!=',
  317. '||': 'or',
  318. '&&': 'and',
  319. '?': 'and', # h4x
  320. ':': 'or', # ^
  321. '.': '+',
  322. }
  323. def __init__(self, parent):
  324. # not sure if needed
  325. self.parent = parent
  326. self.loops = None
  327. def handle_item(self, type, value):
  328. if type == 'string':
  329. return value.decode('string-escape')
  330. elif type == 'html':
  331. return value
  332. elif type == 'include':
  333. return self.TAGS['include'] % self.handle_include(value)
  334. elif type == 'const':
  335. return self.TAGS['include'] % (value.lower() + '.html')
  336. elif type == 'abbrtext':
  337. if value.startswith('"'):
  338. value = remove_backslashes(value)
  339. return self.TAGS['filter'] % ('reverse_format(strings.ABBRTEXT)',
  340. value.strip('\'"'))
  341. return value
  342. def handle_include(self, value):
  343. value = value.replace(HTDOCS_HARDCODED_PATH, '')
  344. if value in RENAME:
  345. value = RENAME[value]
  346. return value
  347. def translate_expression(self, exp, tagname, loops):
  348. mode = None
  349. if tagname == 'loop':
  350. mode = 'loop'
  351. self.loops = loops
  352. result = self._translate_expression(exp, mode=mode)
  353. if LOOP_TAG_DEBUG and loops:
  354. print " > exp(%s) :: %s" % (', '.join(loops), result)
  355. if EXPRESSION_TRANSLATOR_DEBUG:
  356. print "->", repr(result)
  357. return result
  358. def _translate_expression(self, exp, mode=None):
  359. parts = []
  360. result = []
  361. for type, value in exp:
  362. if type == 'option':
  363. value = 'board.options.%s' % value
  364. elif type == 'path':
  365. value = 'board.name'
  366. elif type == 'advinclude':
  367. raise AdvInclude(value)
  368. elif type == 'function':
  369. name, subexp = value
  370. if name in RENAME:
  371. name = RENAME[name]
  372. parsed = self._translate_expression(subexp, mode='function')
  373. if name == 'void':
  374. value = '(%s)' % ', '.join(parsed)
  375. elif len(parsed) > 1:
  376. value = '(%s)|%s(%s)'\
  377. % (parsed[0], name, ', '.join(parsed[1:]))
  378. elif len(parsed) == 1 and ''.join(parsed):
  379. value = '(%s)|%s' % (parsed[0], name)
  380. else:
  381. value = '%s()' % name
  382. if VARIABLES_DEBUG and name != 'void':
  383. print " filter", name
  384. elif type == 'var':
  385. if value in RENAME:
  386. value = RENAME[value]
  387. for loop in self.loops[::-1]:
  388. if loop in KNOWN_LOOPS and value in KNOWN_LOOPS[loop][1]:
  389. value = '%s.%s' % (KNOWN_LOOPS[loop][0], value)
  390. if VARIABLES_DEBUG:
  391. print " var", value
  392. elif type == 'const':
  393. if value in RENAME:
  394. value = RENAME[value]
  395. if value.startswith("S_"):
  396. value = 'strings.%s' % value[2:]
  397. elif config and hasattr(config, value):
  398. value = 'config.%s' % value
  399. if VARIABLES_DEBUG:
  400. print " const", value
  401. elif type == 'regex':
  402. do_lower = value.endswith('i')
  403. action = value.startswith('/^') and 'startswith' or 'count'
  404. value = value.strip('/i^')
  405. variable = result.pop()
  406. if variable == 'not':
  407. variable = result.pop()
  408. result.append('not')
  409. result.append('%s.%s("%s")' % (variable, action, value))
  410. value = None
  411. elif type == 'operator':
  412. value = self.OPERATORS.get(value, value)
  413. elif type == 'comma':
  414. parts.append(result)
  415. result = []
  416. value = None
  417. if value:
  418. result.append(value)
  419. if mode == 'function':
  420. parts.append(result)
  421. return [' '.join(x) for x in parts]
  422. elif mode == 'loop':
  423. itervarname = 'i'
  424. if len(exp) == 1:
  425. type, value = exp[0]
  426. if type in ('var', 'const'):
  427. if value in KNOWN_LOOPS:
  428. itervarname = KNOWN_LOOPS[value][0]
  429. elif value.lower().endswith('s'):
  430. itervarname = value.lower().rstrip('s')
  431. else:
  432. itervarname = value.lower() + '_item'
  433. return (itervarname, ' '.join(result))
  434. else:
  435. return ' '.join(result)
  436. class AdvInclude(Exception):
  437. '''This is not an exception but an exceptional condition
  438. Advincludes are complete includes with template tags parsing
  439. and everything, but inside a <var> tag, so the most sensible
  440. way to handle them was to raise an exception'''
  441. def __init__(self, value):
  442. self.value = value
  443. def template_filename(constname):
  444. return os.path.join(TEMPLATES_DIR, '%s.html' % constname.lower())
  445. def main():
  446. parser = optparse.OptionParser()
  447. parser.add_option("-f", "--filename", default="futaba_style.pl",
  448. help="Location of the futaba_style.pl file")
  449. parser.add_option("-o", "--only", default=None, metavar="CONST",
  450. help="Parse only one constant in futaba_style.pl")
  451. parser.add_option("-n", "--dry-run", action="store_true",
  452. help="Don't write templates to disk")
  453. group = optparse.OptionGroup(parser, "Debug channels")
  454. group.add_option("--futaba-style-debug", action="store_true")
  455. group.add_option("--expression-debug", action="store_true")
  456. group.add_option("--translator-debug", action="store_true")
  457. group.add_option("--loop-debug", action="store_true")
  458. group.add_option("--variables-debug", action="store_true")
  459. parser.add_option_group(group)
  460. (options, args) = parser.parse_args()
  461. # set debug channels. oh god h4x
  462. global FUTABA_STYLE_DEBUG, EXPRESSION_DEBUG, EXPRESSION_TRANSLATOR_DEBUG
  463. global LOOP_TAG_DEBUG, VARIABLES_DEBUG
  464. FUTABA_STYLE_DEBUG = options.futaba_style_debug
  465. EXPRESSION_DEBUG = options.expression_debug
  466. EXPRESSION_TRANSLATOR_DEBUG = options.translator_debug
  467. LOOP_TAG_DEBUG = options.loop_debug
  468. VARIABLES_DEBUG = options.variables_debug
  469. FutabaStyleParser(filename=options.filename,
  470. only=options.only,
  471. dry_run=options.dry_run)
  472. if __name__ == '__main__':
  473. main()