From fb398839f60db29a078f3cf8b720585d9c2513e9 Mon Sep 17 00:00:00 2001 From: "York.Gu" Date: Wed, 27 Apr 2016 11:32:47 +0800 Subject: [PATCH 1/4] improve performance --- IP.java | 51 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/IP.java b/IP.java index d0b3a81..912cd40 100644 --- a/IP.java +++ b/IP.java @@ -1,6 +1,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Arrays; @@ -72,6 +73,26 @@ public static void load(String filename, boolean strict) throws Exception { } } + private static int quickFindInIndexBuffer(long ip2long_value, int start, int end) { + int middle = (start + end) / 2; + middle = middle & 0xFFFFFFF8; + if (middle <= start) return end; + long middleValue = int2long(indexBuffer.getInt(middle)); + if (middleValue == ip2long_value) { + return middle; + } else if (middleValue > ip2long_value) { + if (middle + 8 == end) { + return middle; + } + return quickFindInIndexBuffer(ip2long_value, start, middle); + } else { + if (middle + 8 == end) { + return end; + } + return quickFindInIndexBuffer(ip2long_value, middle, end); + } + } + public static String[] find(String ip) { int ip_prefix_value = new Integer(ip.substring(0, ip.indexOf("."))); long ip2long_value = ip2long(ip); @@ -80,13 +101,15 @@ public static String[] find(String ip) { long index_offset = -1; int index_length = -1; byte b = 0; - for (start = start * 8 + 1024; start < max_comp_len; start += 8) { - if (int2long(indexBuffer.getInt(start)) >= ip2long_value) { - index_offset = bytesToLong(b, indexBuffer.get(start + 6), indexBuffer.get(start + 5), indexBuffer.get(start + 4)); - index_length = 0xFF & indexBuffer.get(start + 7); - break; - } + + int index = start * 8 + 1024; + if (int2long(indexBuffer.getInt(index)) < ip2long_value) { + index = quickFindInIndexBuffer(ip2long_value, index, max_comp_len); } + index_offset = bytesToLong(b, indexBuffer.get(index + 6), indexBuffer.get(index + 5), indexBuffer.get(index + 4)); + index_length = 0xFF & indexBuffer.get(index + 7); + if (index_length <= 0) + return new String[0]; byte[] areaBytes; @@ -160,13 +183,15 @@ private static long bytesToLong(byte a, byte b, byte c, byte d) { } private static int str2Ip(String ip) { - String[] ss = ip.split("\\."); - int a, b, c, d; - a = Integer.parseInt(ss[0]); - b = Integer.parseInt(ss[1]); - c = Integer.parseInt(ss[2]); - d = Integer.parseInt(ss[3]); - return (a << 24) | (b << 16) | (c << 8) | d; + try { + byte[] bytes = java.net.InetAddress.getByName(ip).getAddress(); + + return ((bytes[0] & 0xFF) << 24) | + ((bytes[1] & 0xFF) << 16) | + ((bytes[2] & 0xFF) << 8) | + bytes[3]; + } catch (UnknownHostException e) {} + return 0; } private static long ip2long(String ip) { From 40317abf9d5f419358d1eeb97282339d242a0519 Mon Sep 17 00:00:00 2001 From: "York.Gu" Date: Thu, 28 Apr 2016 11:10:25 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=80=A7=E8=83=BD?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E7=A8=8B=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + IPOld.java | 183 ++++++++++++++++++++++++++++++++++++++++++++++++++ TestMain.java | 56 +++++++++++++++ 3 files changed, 240 insertions(+) create mode 100644 .gitignore create mode 100644 IPOld.java create mode 100644 TestMain.java diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6b468b6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.class diff --git a/IPOld.java b/IPOld.java new file mode 100644 index 0000000..4a2ea25 --- /dev/null +++ b/IPOld.java @@ -0,0 +1,183 @@ +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.nio.charset.Charset; +import java.util.Random; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +class IPOld { + + public static String randomIp() { + Random r = new Random(); + StringBuffer str = new StringBuffer(); + str.append(r.nextInt(1000000) % 255); + str.append("."); + str.append(r.nextInt(1000000) % 255); + str.append("."); + str.append(r.nextInt(1000000) % 255); + str.append("."); + str.append(0); + + return str.toString(); + } + + public static void main(String[] args){ + IP.load("H:\\loveapp\\codebase\\17mon\\17monipdb.dat"); + + Long st = System.nanoTime(); + for (int i = 0; i < 1000000; i++) + { + IP.find(randomIp()); + } + Long et = System.nanoTime(); + System.out.println((et - st) / 1000 / 1000); + + System.out.println(Arrays.toString(IP.find("118.28.8.8"))); + } + + public static boolean enableFileWatch = false; + + private static int offset; + private static int[] index = new int[256]; + private static ByteBuffer dataBuffer; + private static ByteBuffer indexBuffer; + private static Long lastModifyTime = 0L; + private static File ipFile ; + private static ReentrantLock lock = new ReentrantLock(); + + public static void load(String filename) { + ipFile = new File(filename); + load(); + if (enableFileWatch) { + watch(); + } + } + + public static void load(String filename, boolean strict) throws Exception { + ipFile = new File(filename); + if (strict) { + int contentLength = Long.valueOf(ipFile.length()).intValue(); + if (contentLength < 512 * 1024) { + throw new Exception("ip data file error."); + } + } + load(); + if (enableFileWatch) { + watch(); + } + } + + public static String[] find(String ip) { + int ip_prefix_value = new Integer(ip.substring(0, ip.indexOf("."))); + long ip2long_value = ip2long(ip); + int start = index[ip_prefix_value]; + int max_comp_len = offset - 1028; + long index_offset = -1; + int index_length = -1; + byte b = 0; + for (start = start * 8 + 1024; start < max_comp_len; start += 8) { + if (int2long(indexBuffer.getInt(start)) >= ip2long_value) { + index_offset = bytesToLong(b, indexBuffer.get(start + 6), indexBuffer.get(start + 5), indexBuffer.get(start + 4)); + index_length = 0xFF & indexBuffer.get(start + 7); + break; + } + } + + byte[] areaBytes; + + lock.lock(); + try { + dataBuffer.position(offset + (int) index_offset - 1024); + areaBytes = new byte[index_length]; + dataBuffer.get(areaBytes, 0, index_length); + } finally { + lock.unlock(); + } + + return new String(areaBytes, Charset.forName("UTF-8")).split("\t", -1); + } + + private static void watch() { + Executors.newScheduledThreadPool(1).scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + long time = ipFile.lastModified(); + if (time > lastModifyTime) { + lastModifyTime = time; + load(); + } + } + }, 1000L, 5000L, TimeUnit.MILLISECONDS); + } + + private static void load() { + lastModifyTime = ipFile.lastModified(); + FileInputStream fin = null; + lock.lock(); + try { + dataBuffer = ByteBuffer.allocate(Long.valueOf(ipFile.length()).intValue()); + fin = new FileInputStream(ipFile); + int readBytesLength; + byte[] chunk = new byte[4096]; + while (fin.available() > 0) { + readBytesLength = fin.read(chunk); + dataBuffer.put(chunk, 0, readBytesLength); + } + dataBuffer.position(0); + int indexLength = dataBuffer.getInt(); + byte[] indexBytes = new byte[indexLength]; + dataBuffer.get(indexBytes, 0, indexLength - 4); + indexBuffer = ByteBuffer.wrap(indexBytes); + indexBuffer.order(ByteOrder.LITTLE_ENDIAN); + offset = indexLength; + + int loop = 0; + while (loop++ < 256) { + index[loop - 1] = indexBuffer.getInt(); + } + indexBuffer.order(ByteOrder.BIG_ENDIAN); + } catch (IOException ioe) { + ioe.printStackTrace(); + } finally { + try { + if (fin != null) { + fin.close(); + } + } catch (IOException e){ + e.printStackTrace(); + } + lock.unlock(); + } + } + + private static long bytesToLong(byte a, byte b, byte c, byte d) { + return int2long((((a & 0xff) << 24) | ((b & 0xff) << 16) | ((c & 0xff) << 8) | (d & 0xff))); + } + + private static int str2Ip(String ip) { + String[] ss = ip.split("\\."); + int a, b, c, d; + a = Integer.parseInt(ss[0]); + b = Integer.parseInt(ss[1]); + c = Integer.parseInt(ss[2]); + d = Integer.parseInt(ss[3]); + return (a << 24) | (b << 16) | (c << 8) | d; + } + + private static long ip2long(String ip) { + return int2long(str2Ip(ip)); + } + + private static long int2long(int i) { + long l = i & 0x7fffffffL; + if (i < 0) { + l |= 0x080000000L; + } + return l; + } +} diff --git a/TestMain.java b/TestMain.java new file mode 100644 index 0000000..384f494 --- /dev/null +++ b/TestMain.java @@ -0,0 +1,56 @@ +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.nio.charset.Charset; +import java.util.Random; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + + +class TestMain { + + public static void main(String[] args){ + IPOld.load("/Users/york/htdocs/kankan-mp4-extractor/libs/ipip/17monipdb.dat"); + IP.load("/Users/york/htdocs/kankan-mp4-extractor/libs/ipip/17monipdb.dat"); + + final int testSize = 100 * 10000; + String[] ips = new String[testSize]; + for(int i=0; i Date: Thu, 28 Apr 2016 22:39:53 +0800 Subject: [PATCH 3/4] improve performance and remove recursion function call --- IP.java | 35 ++++++++++++++++++++--------------- TestMain.java | 1 + 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/IP.java b/IP.java index 912cd40..bcefc3d 100644 --- a/IP.java +++ b/IP.java @@ -74,30 +74,35 @@ public static void load(String filename, boolean strict) throws Exception { } private static int quickFindInIndexBuffer(long ip2long_value, int start, int end) { - int middle = (start + end) / 2; - middle = middle & 0xFFFFFFF8; - if (middle <= start) return end; - long middleValue = int2long(indexBuffer.getInt(middle)); - if (middleValue == ip2long_value) { - return middle; - } else if (middleValue > ip2long_value) { - if (middle + 8 == end) { + while(true){ + int middle = (start + end) / 2; + middle = middle & 0xFFFFFFF8; + if (middle <= start) return end; + long middleValue = int2long(indexBuffer.getInt(middle)); + if (middleValue == ip2long_value) { return middle; + } else if (middleValue > ip2long_value) { + if (middle + 8 == end) { + return middle; + } + end = middle; + } else { + if (middle + 8 == end) { + return end; + } + start = middle; } - return quickFindInIndexBuffer(ip2long_value, start, middle); - } else { - if (middle + 8 == end) { - return end; - } - return quickFindInIndexBuffer(ip2long_value, middle, end); } } public static String[] find(String ip) { - int ip_prefix_value = new Integer(ip.substring(0, ip.indexOf("."))); long ip2long_value = ip2long(ip); + int ip_prefix_value = ((int)ip2long_value >> 24) & 0xFF; int start = index[ip_prefix_value]; int max_comp_len = offset - 1028; + if(ip_prefix_value < index.length-1) { + max_comp_len = index[ip_prefix_value+1] * 8 + 1024; + } long index_offset = -1; int index_length = -1; byte b = 0; diff --git a/TestMain.java b/TestMain.java index 384f494..99ec539 100644 --- a/TestMain.java +++ b/TestMain.java @@ -22,6 +22,7 @@ public static void main(String[] args){ for(int i=0; i Date: Wed, 11 May 2016 17:55:38 +0800 Subject: [PATCH 4/4] bug fix: str2Ip value is not correct under some ips --- IP.java | 17 +++++++---------- TestMain.java | 4 ++++ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/IP.java b/IP.java index bcefc3d..6cbfca0 100644 --- a/IP.java +++ b/IP.java @@ -1,7 +1,6 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; -import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Arrays; @@ -188,15 +187,13 @@ private static long bytesToLong(byte a, byte b, byte c, byte d) { } private static int str2Ip(String ip) { - try { - byte[] bytes = java.net.InetAddress.getByName(ip).getAddress(); - - return ((bytes[0] & 0xFF) << 24) | - ((bytes[1] & 0xFF) << 16) | - ((bytes[2] & 0xFF) << 8) | - bytes[3]; - } catch (UnknownHostException e) {} - return 0; + String[] ss = ip.split("\\."); + int a, b, c, d; + a = Integer.parseInt(ss[0]); + b = Integer.parseInt(ss[1]); + c = Integer.parseInt(ss[2]); + d = Integer.parseInt(ss[3]); + return (a << 24) | (b << 16) | (c << 8) | d; } private static long ip2long(String ip) { diff --git a/TestMain.java b/TestMain.java index 99ec539..b815a26 100644 --- a/TestMain.java +++ b/TestMain.java @@ -22,6 +22,10 @@ public static void main(String[] args){ for(int i=0; i