python3 bytes problem fix + better unicode support + unicode tests

bin/binaries/lib/python/microcule/__init__.py

@@ -14,19 +14,19 @@ def format(self, record):
         record = record.__dict__.copy()
         record['exc_info'] = None
         try:
-            json.dumps(record['args'])
+            json.dumps(record['args'], ensure_ascii=True)
         except Exception:
             del record['args']
             record['msg'] = record['message']
         res = {'type': 'log', 'payload': {'entry': record}}
-        return json.dumps(res)
+        return json.dumps(res, ensure_ascii=True)
 
 
 class SimpleMicroculeJSONFormatter(logging.Formatter):
     def format(self, record):
         msg = logging.Formatter.format(self, record)
         res = {'type': 'log', 'payload': {'entry': msg}}
-        return json.dumps(res)
+        return json.dumps(res, ensure_ascii=True)
 
 
 class MicroculeExceptHook:
@@ -48,15 +48,31 @@ def send_exception(self, info=None):
             code = info[0].__module__ + '.' + code
         payload = {'code': code}
         if hasattr(info[1], 'args'):
-            payload['args'] = repr(info[1].args)
+            try:
+                payload['args'] = json.dumps(info[1].args, ensure_ascii=True)
+            except Exception:
+                payload['args'] = '!'+repr(info[1].args)
         if self.verbose:
             payload['error'] = ''.join(traceback.format_exception(*info))
         else:
             payload['error'] = str(info[1])
         res = {'type': 'error', 'payload': payload}
-        if isinstance(info[1], ImportError) and info[1].message.startswith('No module named '):
-            payload['code'] = 'MODULE_NOT_FOUND'
-            payload['module'] = info[1].message.replace('No module named ', '')
+        if isinstance(info[1], ImportError):
+            name = None
+            if hasattr(info[1],'name'):
+                name = info[1].name
+            elif str(info[1]).startswith('No module named '):
+                name = str(info[1]).replace('No module named ', '')
+                if name[:1] == name[-1:] == "'":
+                    name = name[1:-1]
+            if name:
+                payload['code'] = 'MODULE_NOT_FOUND'
+                payload['module'] = name
+                error = '%s(%s): %s' % (payload['code'], code, str(info[1]))
+                if self.verbose:
+                    payload['error'] += error
+                else:
+                    payload['error'] = error
         if isinstance(info[1], (pkg_resources.VersionConflict, pkg_resources.DistributionNotFound)):
             req = None
             try:
@@ -78,30 +94,33 @@ def send_exception(self, info=None):
                     payload['error'] += error
                 else:
                     payload['error'] = error
-        sys.stderr.write(json.dumps(res)+'\n')
+        sys.stderr.write(json.dumps(res, ensure_ascii=True)+'\n')
         sys.stderr.flush()
-        sys.stderr.write(json.dumps({'type': 'statusCode', 'payload': {'value': 500}})+'\n')
+        sys.stderr.write(json.dumps({'type': 'statusCode', 'payload': {'value': 500}}, ensure_ascii=True)+'\n')
         sys.stderr.flush()
         if self.display:
             sys.stdout.write(payload['error'].rstrip('\n')+'\n')
             sys.stdout.flush()
-        sys.stderr.write(json.dumps({'type': 'end'})+'\n')
+        sys.stderr.write(json.dumps({'type': 'end'}, ensure_ascii=True)+'\n')
         sys.stderr.flush()
 
 
 class wsgi(wsgiref.handlers.CGIHandler):
     def __init__(self, Hook=None):
         self.Hook = Hook or getattr(sys.modules.get('__main__',sys), 'Hook', None)
+        stdout = sys.stdout
+        if hasattr(stdout, 'buffer'):
+            stdout = stdout.buffer
         wsgiref.handlers.BaseCGIHandler.__init__(
-            self, sys.stdin, sys.stdout, sys.stderr, {},
+            self, sys.stdin, stdout, sys.stderr, {},
             multithread=False, multiprocess=True
         )
 
     def send_headers(self):
         self.cleanup_headers()
         self.headers_sent = True
         head = {'type': 'writeHead', 'payload': {'code': self.status, 'headers': dict(self.headers)}}
-        sys.stderr.write(json.dumps(head)+'\n')
+        sys.stderr.write(json.dumps(head, ensure_ascii=True)+'\n')
         sys.stderr.flush()
 
     def add_cgi_vars(self):
@@ -174,7 +193,10 @@ def parse_argv(argv=None):
     # print(argv[5]) # -s
     # print(argv[6]) # the service
     # TODO: do something with service variable
-    service = argv[6]
+    service = json.loads(argv[6])
+
+    if 'file' in service:
+      code = compile(code, service['file'], 'exec')
 
     debug_output = 'gateway' in Hook['resource'].get('name', 'gateway')
     prod_mode = Hook['resource'].get('mode', 'Debug') == 'Production'

bin/binaries/micro-python

@@ -9,6 +9,14 @@ def __prepare():
     # print(sys.argv)
     # print(sys.path)
 
+    if hasattr(sys.stdout, 'encoding') and not sys.stdout.encoding:
+        sys.stdout.flush()
+        import codecs
+        out = sys.stdout
+        sys.stdout = codecs.getwriter('UTF-8')(out, 'backslashreplace')
+        sys.stdout.encoding = 'UTF-8'
+        sys.stdout.buffer = out
+
     import microcule
 
     code, service, Hook = microcule.parse_argv()

examples/services/echo/echo-py3.py

@@ -1,20 +1,11 @@
-import pprint
-import logging
-import microcule
-
-log = logging.getLogger('echo-py3')
-
-
-def app(environ, start_response):
-    start_response('200 OK', [('content-type', 'text/plain')])
-    res = ["Hello, this is a Python script."]
-    res.append("Hook['params'] is populated with request parameters")
-    res.append(pprint.pformat(Hook['params']))
-    res.append("Hook['req'] is the http request")
-    res.append(pprint.pformat['req']['url']))
-    log.info('hello logs')
-    log.warn('%s', Hook['params'])
-    return '\n'.join(res)
-
-if __name__ == '__main__':
-    microcule.wsgi(Hook).run(app)
+# -*- coding: utf-8 -*-
+print("Hello, this is a Python script.")
+print("Hook['params'] is populated with request parameters")
+print(Hook['params'])
+print("Hook['req'] is the http request")
+print(Hook['req']['url'])
+print(u"проверка кодировки")
+print(u"סיסטעם פּרובירן")
+print(u"編碼測試")
+print(u"エンコードテスト")
+print(u"एन्कोडिंग परीक्षण")

examples/services/echo/echo-wsgi.py

@@ -1,6 +1,9 @@
+# -*- coding: utf-8 -*-
+import sys
 import pprint
 import logging
 import microcule
+from six import u
 
 log = logging.getLogger('echo-py')
 
@@ -11,9 +14,19 @@ def app(environ, start_response):
     res.append(pprint.pformat(Hook['params']))
     res.append("Hook['req'] is the http request")
     res.append(pprint.pformat(Hook['req']['url']))
+    res.append(u"проверка кодировки")
+    res.append(u"סיסטעם פּרובירן")
+    res.append(u"編碼測試")
+    res.append(u"エンコードテスト")
+    res.append(u"एन्कोडिंग परीक्षण")
+    #print(repr(res))
+    res = u'\n'.join(res)
+    #print(repr(res))
+    res = res.encode('utf-8')
+    #print(repr(res))
     log.info('hello logs')
     log.warn('%s', Hook['params'])
-    return '\n'.join(res)
+    return [res]
 
 if __name__ == '__main__':
-    microcule.wsgi(Hook).run(app)
+    microcule.wsgi(Hook).run(app)

examples/services/echo/echo.py

@@ -1,8 +1,17 @@
-import pprint, sys, json
-print "Hello, this is a Python script."
-print "Hook['params'] is populated with request parameters"
+# -*- coding: utf-8 -*-
+import pprint, sys, codecs, json, logging
+log = logging.getLogger('').debug
+print("Hello, this is a Python script.")
+print("Hook['params'] is populated with request parameters")
 pprint.pprint(Hook['params'])
-print "Hook['env'] is populated with env variables"
-pprint.pprint(Hook['env'])
+#print("Hook['env'] is populated with env variables")
+#pprint.pprint(Hook['env'])
+print("Hook['req'] is the http request")
+print(Hook['req']['url'])
+print(u"проверка кодировки")
+print(u"סיסטעם פּרובירן")
+print(u"編碼測試")
+print(u"エンコードテスト")
+print(u"एन्कोडिंग परीक्षण")
 log('hello logs')
-log(Hook['params'])
+log(repr(Hook['params']))