Test bad references

Dockerfile

@@ -19,4 +19,6 @@ COPY tools/RK211.COM .
 COPY tools/RUSSIAN.RK .
 COPY tools/8X16.RK .
 
+COPY tools/DEBUG.EXE .
+
 ENTRYPOINT ["python3", "-m", "unittest", "-v"]

docker-compose.yml

@@ -9,10 +9,11 @@ services:
   test:
     build: .
     pull_policy: build
+    ipc: host # FIXME
     environment:
     - DISPLAY
     volumes:
-    - "/tmp/.X11-unix:/tmp/.X11-unix"
-    - "/dev/snd:/dev/snd"
+    - /tmp/.X11-unix:/tmp/.X11-unix
+    - /dev/snd:/dev/snd
     depends_on:
     - testbox

patches/bad-references.json

@@ -14,6 +14,13 @@
             38281
         ]
     },
+    {
+        "source": "GAME.EXE",
+        "offset": 198944,
+        "bad-references": [
+            212818
+        ]
+    },
     {
         "source": "U.EXE",
         "offset": 92153,

tests/__init__.py

@@ -5,6 +5,9 @@
 
 import testbox
 
+import tools.file
+import tools.lzw
+
 
 class TestCase(unittest.TestCase):
     def __init__(self, *args, **kwargs):
@@ -48,10 +51,22 @@ def setUp(self):
                 box.send_keys(keys)
                 time.sleep(0.1)
 
-            box.wait_image('screenshots/INSTALL/install.png', bbox=(0, 16, 640, 400), timeout=1)
+            box.wait_image('screenshots/INSTALL/install.png', bbox=(0, 16, 640, 400), timeout=2)
             self.assertTrue(os.path.isfile('CONFIG.U6'))
 
     def tearDown(self):
         os.remove('MAP')
         os.remove('CONFIG.U6')
         shutil.rmtree('SAVEGAME')
+
+
+class ConfiguredTestCase(unittest.TestCase):
+    def setUp(self):
+        tools.file.write('CONFIG.U6', b'\x76\x6d\x61\x33\x38\x38\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00')
+        tools.file.write('MAP', tools.lzw.decompress(tools.file.read('original/LZMAP')))
+
+    def tearDown(self):
+        os.remove('MAP')
+        os.remove('CONFIG.U6')
+        if os.path.isdir('SAVEGAME'):
+            shutil.rmtree('SAVEGAME')

tests/test_game.py

@@ -1,14 +1,16 @@
+import shutil
 import time
 import unittest
 
 import testbox
 import tests
 
 
-class TestGame(tests.InstalledTestCase):
+class TestGame(tests.ConfiguredTestCase):
     @unittest.skip('TBD')
     def testGame(self):
         # FIXME игре нужен созданный персонаж
+        shutil.copytree('tests/testgame', 'SAVEGAME')
 
         configuration = {
             'cpu': {

tests/test_u.py

@@ -14,21 +14,21 @@ def testExitWithDriver(self):
             },
             'render': {
                 'scaler': 'normal5x',
-            },
+            }
         }
 
         with testbox.TestBox(
             [
                 '-c', 'MOUNT C: .',
                 '-c', 'C:',
                 '-c', 'RK211.COM /L:RUSSIAN.RK /F:8X16.RK', # FIXME `RK.COM`
-                'U.EXE',
+                'U.EXE'
             ],
             configuration,
             timeout_error=AssertionError,
         ) as box:
             box.send_keys('\x1b') # Esc.
-            box.wait_image('screenshots/U/menu.png', timeout=6)
+            box.wait_image('screenshots/U/menu.png', timeout=10)
             box.send_keys('\x11') # Ctrl+Q.
             box.wait_image('screenshots/U/exit.png', timeout=1)
 
@@ -40,17 +40,46 @@ def testU(self):
             },
             'render': {
                 'scaler': 'normal5x',
-            },
+            }
         }
 
         with testbox.TestBox(
             [
                 '-c', 'MOUNT C: .',
                 '-c', 'C:',
-                '-c', 'RK.COM /L:RUSSIAN.RK /F:8X16.RK',
-                'U.EXE',
+                '-c', 'RK211.COM /L:RUSSIAN.RK /F:8X16.RK', # FIXME `RK.COM`
+                'U.EXE'
             ],
             configuration,
             timeout_error=AssertionError,
         ) as box:
+            box.send_keys('\x1b') # Esc.
+            box.wait_image('screenshots/U/menu.png', timeout=10)
+            box.send_keys('▾')
+            box.wait_image('screenshots/U/new.png', timeout=1)
+            time.sleep(0.1)
+            box.send_keys('\r')
+            box.wait_image('screenshots/U/name.png', timeout=10)
+            box.send_keys('Vladimir\r')
+            time.sleep(0.1)
+            box.send_keys('M')
+            time.sleep(0.1)
+            box.send_keys('C')
+            box.wait_image('screenshots/U/create.png', timeout=10)
+            box.send_keys('\r')
+            box.wait_image('screenshots/U/scroll.png', timeout=10)
+            box.send_keys('\r')
+            time.sleep(0.1)
+            box.send_keys('\r')
+            box.wait_image('screenshots/U/inside.png', timeout=5)
+            box.send_keys('\r')
+            time.sleep(0.1)
+            box.send_keys('\r')
+            for _ in range(7):
+                time.sleep(0.5)
+                box.send_keys('A')
+
+            # TODO Five spaces one Esc
+
+            print('Go')
             time.sleep(1000)

tools/duplicates.py

@@ -1,5 +1,7 @@
 import json
 import os
+import random
+import sys
 
 
 output_directory = os.getcwd()
@@ -9,14 +11,46 @@
     references = {(x['source'], x['offset']): x['references'] for x in json.loads(f.read())}
 
 with open('patches/bad-references.json') as f:
-    for x in json.loads(f.read()):
-        bad = set(x['bad-references'])
-        references[(x['source'], x['offset'])] = [
-            y
-            for y in references[(x['source'], x['offset'])]
-            if y['origin'] not in bad
-        ]
+    br = json.loads(f.read())
+
+for x in br:
+    bad = set(x['bad-references'])
+    references[(x['source'], x['offset'])] = [
+        y
+        for y in references[(x['source'], x['offset'])]
+        if y['origin'] not in bad
+    ]
 
 for k, v in references.items():
     if len(v) > 1:
         print('WARNING', k[0], hex(k[1]), len(v), 'references:', ', '.join((hex(x['origin']) for x in v)))
+
+if sys.argv[1:] in (['--fix'], ['--fix-half'], ['--fix-tenth']):
+    added = set()
+    if sys.argv[1:] == ['--fix-half']:
+        threshold = 0.5
+    elif sys.argv[1:] == ['--fix-tenth']:
+        threshold = 0.1
+    else:
+        threshold = 1.0
+
+    for k, v in references.items():
+        if len(v) > 1:
+            for b in br:
+                if b['source'] == k[0] and b['offset'] == k[1]:
+                    found = b
+                    break
+            else:
+                found = {'source': k[0], 'offset': k[1], 'bad-references': [x['origin'] for x in v]}
+                br.append(found)
+
+            for x in v:
+                if random.random() <= threshold:
+                    found['bad-references'].append(x['origin'])
+                    added.add((k[0], k[1], x['origin']))
+            found['bad-references'] = sorted(found['bad-references'])
+
+    print('Added:', sorted(added))
+    br = sorted(br, key=lambda x: (x['source'], x['offset'])) 
+    with open('patches/bad-references.json', 'w') as f:
+        f.write(json.dumps(br, indent=4))

tools/patch.py

@@ -232,34 +232,49 @@ def pi_jump(s, a):
     elif binary == 'U.EXE':
         patches.patch_U(d)
 
+    ds = d[header + segments[-1]*0x10:]
+
     uninitialized_fill, = find_all(d, [0xbf, None, None, 0xb9, None, None, 0x2b, 0xcf, 0xf3, 0xaa])
     initialized_size = int.from_bytes(d[uninitialized_fill+1:uninitialized_fill+3], 'little')
     assert initialized_size in (len(d) - segments[-1]*0x10 - header, len(d) - segments[-1]*0x10 - header-2)
     uninitialized_size = int.from_bytes(d[uninitialized_fill+4:uninitialized_fill+6], 'little') - initialized_size
+
+    # FIXME not used
     dereferencing = [b'\xbf', b'\xa1', b'\x8b\x1e', b'\xa3', b'\xc4\x1e', b'\xff\x36', b'\x89\x1e', b'\x8c\x06', b'\x8b\x16', b'\x89\x16', b'\x3b\x06', b'\x29\x06', b'\x3b\x16', b'\x19\x16']
+
     system_break_shift, = find_all(d, [0x8b, 0x56, 8, 3, 6, None, None, 0x83, 0xd2, 0, 0x8b, 0xc8, 0x81, 0xc1, 0, 1, 0x83, 0xd2, 0])
     system_break_address = int.from_bytes(d[system_break_shift+5:system_break_shift+7], 'little')
     system_break = int.from_bytes(d[system_break_address+segments[-1]*0x10+header:system_break_address+segments[-1]*0x10+header+2], 'little')
     assert system_break == initialized_size + uninitialized_size
 
+    # FIXME not used
+    extra_alloc_update, = find_all(d, [0x8b, 0x3e, None, None, 0x81, 0xff, 0, 2, 0x73, 7, 0xbf, 0, 2, 0x89, 0x3e])
+    extra_alloc_address = int.from_bytes(d[extra_alloc_update+2:extra_alloc_update+4], 'little')
+    extra_alloc = int.from_bytes(ds[extra_alloc_address:extra_alloc_address+2], 'little')
+
     replaces = []
-    ds = d[header + segments[-1]*0x10:]
     ds_size = len(ds)
     ds = ljust(ds, system_break, b'www.old-games.ru.')
+    ds_full_size = len(ds)
     required_space = 0
     if mode == 'russian':
         for t in translation:
             if t['source'] == binary:
                 if not t['russian'].startswith('FIXME ') and t['russian'] != t['english']:
                     references_segments = [x['segment'] for x in references.get((binary, t['offset']), [])]
+                    new_string = t['russian'].encode('cp866')
+                    import random
+                    if random.random() < 0.95:
+                        new_string = t['russian'].encode('cp866')[:len(t['english'].encode('cp866'))]
 
-                    if len(t['russian'].encode('cp866')) <= len(t['english'].encode('cp866')):
+                    if len(new_string) <= len(t['english'].encode('cp866')):
                         # FIXME упаковать фразы лучше
-                        message = t['russian'].encode('cp866').ljust(len(t['english']), b'\x00')
+                        message = new_string.ljust(len(t['english']), b'\x00')
                         replaces.append((t['offset'], len(message), message))
                         replaced += 1
 
                     elif t['offset'] > header + segments[-1]*0x10:
+                        # Data segment.
                         # FIXME reuse old space
                         rr = references.get((binary, t['offset']), [])
                         for r in rr:
@@ -272,17 +287,28 @@ def pi_jump(s, a):
                         added += 1
 
                     elif all(map(lambda x: isinstance(x, int), references_segments)):
+                        # (Probably) only referenced by far pointers.
                         print(f'String {repr(t["russian"])} can be moved!')
-                        # FIXME why it can?
 
                 else:
                     missing += 1
 
     ds[system_break_address:system_break_address+2] = len(ds).to_bytes(2, 'little')
+
+    print('Old DS size:', hex(ds_size))
     ds.extend(b'\x00' * ((len(ds) - ds_size + 0x1ff) // 0x200 * 0x200 - len(ds) + ds_size))
+    print('Required space:', hex(len(ds)-ds_full_size))
     assert (len(ds) - ds_size) % 0x200 == 0
     data_space = (len(ds) - ds_size) // 0x200
+    print('DS size:', hex(len(ds)))
     print(f'{binary} — adding {data_space*0x200 + len(code_block)} bytes for {len(functions)} functions and {added} strings')
+
+    # TODO FIXME check for remaining memory
+
+    #Reduce stack???
+    #assert extra_alloc - data_space*0x20 >= 0x200 # Stack? FIXME find a way to leave stack size as is.
+    #ds[extra_alloc_address:extra_alloc_address+2] = (extra_alloc-data_space*0x20).to_bytes(2, 'little')
+
     d[header + segments[-1]*0x10:] = ds
 
     # FIXME переставил это на абзац ниже, проверить, что это ничего не ломает. как будто бы не должно
@@ -330,8 +356,25 @@ def pi_jump(s, a):
     assert ss + 0x10 < 0x10000
     d[0x0e:0x10] = ss.to_bytes(2, 'little')
     d[4:6] = pages.to_bytes(2, 'little')
+
+    assert d[0x0c:0x0e] == b'\xff\xff' # Maximum allocation.
+
     d[0x0a:0x0c] = (max(0, int.from_bytes(d[0x0a:0x0c], 'little') - data_space*0x20)).to_bytes(2, 'little')
-    d[0x0c:0x0e] = (max(0, int.from_bytes(d[0x0c:0x0e], 'little') - data_space*0x20)).to_bytes(2, 'little')
+
+    if d[0x0c:0x0e] != b'\xff\xff':
+        d[0x0c:0x0e] = (max(0, int.from_bytes(d[0x0c:0x0e], 'little') - data_space*0x20)).to_bytes(2, 'little')
+
+    replace_system_breaks = { # TODO more strict.
+        b'\x81\xc7': 1,
+        b'\xb9': 1,
+    }
+    for i in range(len(d)):
+        if int.from_bytes(d[i:i+2], 'little') == system_break:
+            for prefix in replace_system_breaks:
+                if i >= len(prefix) and prefix == d[i-len(prefix):i]:
+                    replace_system_breaks[prefix] -= 1
+                    d[i:i+2] = len(ds).to_bytes(2, 'little')
+    assert set(replace_system_breaks.values()) == {0}
 
     d = d[:header] + code_block + d[header:]