redis-replicator icon indicating copy to clipboard operation
redis-replicator copied to clipboard
verified publisher icon leonchen83

Support getting the raw Redis command

Open tasszz2k opened this issue 1 year ago • 14 comments

Support getting the raw Redis command

Description

  • I'm developing a Redis Real-time Backup using PSYNC. Simply like this: image

  • But I realize there is no way to get the raw command from Event, so I have to switch-case to check each command type and then build the raw command. It's not a nice way to do that.

  • So, I'm looking for a way to get the raw command from Event in Redis. Is there any way to do that?

Sample code:

  • Currently:
public class RealtimeBackup {
    public static void main(String[] args) {
        try {
            Replicator r;
            r = new RedisReplicator("redis://127.0.0.1:6379");
            r.addEventListener((replicator, event) -> {
                handleEvent(event);
            });
            r.open();
        } catch (URISyntaxException | IOException e) {
            throw new RuntimeException(e);
        }
    }

    private static void handleEvent(Event event) {
        // log the raw command to backup
        // System.out.println(event);

        // switch-case instance of the command
        if (event instanceof MSetCommand) {
            MSetCommand mSetCommand = (MSetCommand) event;
            // convert key-value pairs (Map<byte[], byte[]>) to string
            StringBuilder sb = new StringBuilder();
            mSetCommand.getKv().forEach((k, v) -> {
                sb.append(new String(k)).append(" ").append(new String(v)).append(" ");
            });
            System.out.printf("MSET %s\n", sb);
        }
        if (event instanceof SetCommand) {
            SetCommand setCommand = (SetCommand) event;

            String key = new String(setCommand.getKey());
            String value = new String(setCommand.getValue());
            ExistType existType = setCommand.getExistType();

            XATType xatType = setCommand.getXatType();
            Long xatValue = setCommand.getXatValue();
            boolean keepTtl = setCommand.getKeepTtl();

            StringBuilder sb = new StringBuilder();
            sb.append("SET ").append(key).append(" ").append(value).append(" ");

            if (existType != null && existType != ExistType.NONE) {
                sb.append(existType).append(" ");
            }

            // if (get) {
            //     sb.append("GET ").append(" ");
            // }

            if (keepTtl) {
                sb.append("KEEPTTL").append(" ");
            } else if (xatType != null && xatType != XATType.NONE && xatValue != null) {
                sb.append(xatType.name()).append(" ").append(xatValue).append(" ");
            }

            System.out.println(sb.toString().trim());
        }
        // ... and the other commands

    }
}
  • Expected:
public class RealtimeBackup {
    public static void main(String[] args) {
        try {
            Replicator r;
            r = new RedisReplicator("redis://127.0.0.1:6379");
            r.addEventListener((replicator, event) -> {
                handleEvent(event);
            });
            r.open();
        } catch (URISyntaxException | IOException e) {
            throw new RuntimeException(e);
        }
    }

    private static void handleEvent(Event event) {
        // log the raw command to backup
        String rawCommand = event.getRawCommand();
        System.out.println(rawCommand);
    }
}

tasszz2k avatar May 30 '24 07:05 tasszz2k

cc @leonchen83 do you have any ideas for that?

tasszz2k avatar May 30 '24 07:05 tasszz2k 

                Replicator r;
		r = new RedisReplicator("redis://127.0.0.1:6379");
		// ignore PING REPLCONF GETACK
		r.addCommandParser(CommandName.name("PING"), new PingParser());
		r.addCommandParser(CommandName.name("REPLCONF"), new ReplConfParser());
		//
		r.addCommandParser(CommandName.name("APPEND"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("SET"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("SETEX"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("MSET"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("DEL"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("SADD"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("HMSET"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("HSET"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("LSET"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("EXPIRE"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("EXPIREAT"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("GETSET"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("HSETNX"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("MSETNX"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("PSETEX"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("SETNX"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("SETRANGE"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("HDEL"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("LPOP"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("LPUSH"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("LPUSHX"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("LRem"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("RPOP"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("RPUSH"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("RPUSHX"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("ZREM"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("RENAME"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("INCR"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("DECR"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("INCRBY"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("DECRBY"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("PERSIST"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("SELECT"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("FLUSHALL"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("FLUSHDB"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("HINCRBY"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("ZINCRBY"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("MOVE"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("SMOVE"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("PFADD"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("PFCOUNT"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("PFMERGE"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("SDIFFSTORE"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("SINTERSTORE"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("SUNIONSTORE"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("ZADD"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("ZINTERSTORE"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("ZUNIONSTORE"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("BRPOPLPUSH"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("LINSERT"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("RENAMENX"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("RESTORE"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("PEXPIRE"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("PEXPIREAT"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("GEOADD"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("EVAL"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("EVALSHA"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("SCRIPT"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("PUBLISH"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("BITOP"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("BITFIELD"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("SETBIT"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("SREM"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("UNLINK"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("SWAPDB"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("MULTI"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("EXEC"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("ZREMRANGEBYSCORE"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("ZREMRANGEBYRANK"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("ZREMRANGEBYLEX"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("LTRIM"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("SORT"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("RPOPLPUSH"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("ZPOPMIN"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("ZPOPMAX"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("XACK"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("XADD"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("XCLAIM"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("XDEL"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("XGROUP"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("XTRIM"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("XSETID"), new DefaultCommandParser());
		// since redis 6.2
		r.addCommandParser(CommandName.name("COPY"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("LMOVE"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("BLMOVE"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("ZDIFFSTORE"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("GEOSEARCHSTORE"), new DefaultCommandParser());
		// since redis 7.0
		r.addCommandParser(CommandName.name("SPUBLISH"), new DefaultCommandParser());
		r.addCommandParser(CommandName.name("FUNCTION"), new DefaultCommandParser());
		r.addEventListener(new EventListener() {
			@Override
			public void onEvent(Replicator replicator, Event event) {
				if (event instanceof DefaultCommand) {
					DefaultCommand cmd = (DefaultCommand) event;
					System.out.println(cmd.getCommand());
					System.out.println(cmd.getArgs());
				}
			}
		});
		r.open();

leonchen83 avatar May 30 '24 07:05 leonchen83

Oh, let me give it a try. Thank you, @leonchen83

tasszz2k avatar May 30 '24 08:05 tasszz2k

I have another question. I want to start on full sync mode PSYNC ? -1. However, I am confused about handling this event type. Many sub-classes inherit KeyValuePair, while I want to get the raw data.

[main] INFO com.moilioncircle.redis.replicator.RedisSocketReplicator -- REPLCONF ip-address 127.0.0.1
[main] INFO com.moilioncircle.redis.replicator.RedisSocketReplicator -- OK
[main] INFO com.moilioncircle.redis.replicator.RedisSocketReplicator -- REPLCONF capa eof
[main] INFO com.moilioncircle.redis.replicator.RedisSocketReplicator -- OK
[main] INFO com.moilioncircle.redis.replicator.RedisSocketReplicator -- REPLCONF capa psync2
[main] INFO com.moilioncircle.redis.replicator.RedisSocketReplicator -- OK
[main] INFO com.moilioncircle.redis.replicator.RedisSocketReplicator -- PSYNC ? -1

service.Backup -- db: 0, key: test_4314, value: [B@117bcfdc, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_4999, value: [B@73a19967, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_350, value: [B@5e746d37, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_7616, value: [B@6e1b9411, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_6404, value: [B@21d1b321, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_5865, value: [B@5ec46cdd, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_1046, value: [B@2324bfe7, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_2576, value: [B@112d1c8e, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_5808, value: [B@3d49fd31, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_4904, value: [B@4016ccc1, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_106, value: [B@46cb98a3, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_1066, value: [B@3ffb3598, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_7566, value: [B@4da9f723, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_5206, value: [B@3b2f4a93, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_7089, value: [B@213bd3d5, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_4205, value: [B@470a659f, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_3565, value: [B@451882b2, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_4702, value: [B@4a23350, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_6710, value: [B@7b78ed6a, expiredType: NONE, expiredValue: null
service.Backup -- db: 0, key: test_4751, value: [B@6fca5907, expiredType: NONE, expiredValue: null
service.Backup -- db: 2, key: k3, value: [B@6ec65b5e, expiredType: NONE, expiredValue: null
service.Backup -- db: 2, key: k1, value: [B@7bebcd65, expiredType: NONE, expiredValue: null
service.Backup -- db: 2, key: k9, value: [B@47447ccf, expiredType: NONE, expiredValue: null

      this.replicator = new RedisReplicator(redisUri);
      this.replicator.addEventListener(
          (r, event) -> {
            if (event instanceof CustomCommand cmd) {
              handleEvent(cmd);
            } else if (event instanceof KeyValuePair kv) {
              handleKeyValuePair(kv);
            }
          });

image

tasszz2k avatar May 31 '24 04:05 tasszz2k

Hi Using different RdbVisitor will result in different KeyValuePair events for example if you set replicator.setRdbVisitor(new DumpRdbVisitor(replicator)); you will accept DumpKeyValuePair event, this RdbVisitor can parse rdb to redis dump format. you can use redis RESOTRE command directly migrate source date to target redis.

if you set ValueIterableRdbVisitor

replicator.setRdbVisitor(new new ValueIterableRdbVisitor(replicator));
r.addEventListener(new ValueIterableEventListener(new EventListener() {
      @Override
      public void onEvent(Replicator replicator, Event event) {
           if (event instanceof BatchedKeyValuePair<?, ?>) {
               // do something
           }
      }
}));

you will accept BatchedKey**Value**Pair events that can split large key to small chunks

if you don't set any RdbVisitor that equals set DefaultRdbVisitor

you will accept Key**Value**Pair. this can parse rdb to Human-readable information

leonchen83 avatar May 31 '24 04:05 leonchen83

is there any way to parse the raw commands to rdb file format?

When starting the application, I will first dump the data to the dump.rdb file. Then I will watch events and add commands to the incr_x file. After that, I will merge the incr_x file to the dump.rdb file. But I don't know how to do that.

tasszz2k avatar May 31 '24 08:05 tasszz2k

the example incr_1.txt file:

SELECT 0 
APPEND key1 Hello 
SET key2 value1 
SET key3 value2 PXAT 1717143998448 
MSET key4 value3 key5 value4 
HSET hash1 field1 value1 field2 value2 
HSET hash2 field1 value1 
LPUSH list1 initial_value 
LSET list1 0 new_value 
SET key9 new_value 
HSETNX hash3 field value 
MSETNX key10 value1 key11 value2 
SET key12 value PXAT 1717143941725 
SETNX key13 value 
SETRANGE key14 6 Redis 
DEL key12 
LPUSH list3 value1 value2 
RPUSH list7 value1 value2 
INCR counter1 
DECR counter2 
INCRBY counter3 5 
SELECT 1 
FLUSHALL 
FLUSHDB 
HINCRBY hash5 field 5 
ZINCRBY zset4 1 member1

tasszz2k avatar May 31 '24 08:05 tasszz2k

you can use this tool to generate an aof-preamble rdb file. redis can read this file directly. aof-preamble file can split two part. rdb part and aof part

following is cods snapshot

        final OutputStream out = new BufferedOutputStream(new FileOutputStream(new File("/path/to/appendonly.aof")));
        final RawByteListener rawByteListener = new RawByteListener() {
            @Override
            public void handle(byte... rawBytes) {
                try {
                    out.write(rawBytes);
                } catch (IOException ignore) {
                }
            }
        };

        final Replicator replicator = new RedisReplicator("redis://127.0.0.1:6379");
        replicator.setRdbVisitor(new SkipRdbVisitor(replicator));
        
        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
            replicator.removeRawByteListener(rawByteListener);
            Replicators.closeQuietly(replicator);
        }));
        
        replicator.addEventListener(new EventListener() {
            @Override
            public void onEvent(Replicator replicator, Event event) {
                if (event instanceof PreRdbSyncEvent) {
                    replicator.addRawByteListener(rawByteListener);
                }
            }
        });
        replicator.open();

leonchen83 avatar Jun 03 '24 02:06 leonchen83

The code backs up only once. I tried adding some keys, however, when I restored the dump.rdb into another Redis, I didn't see them. That means there are only snapshots, and no incremental :(

tasszz2k avatar Jun 03 '24 10:06 tasszz2k

try change target redis parameters

appendonly yes
aof-use-rdb-preamble yes

and change file name dump.rdb to appendonly.aof

leonchen83 avatar Jun 03 '24 12:06 leonchen83

try change target redis parameters

appendonly yes
aof-use-rdb-preamble yes

and change file name dump.rdb to appendonly.aof

Oh sorry, my fault. I want to note that, the main reason I built an app called redis real-time backup was because I didn't want to enable append-only file. Our application uses Redis as a database (not only for caching), which requires performance and reliability. So we need to back up in real-time without an append-only file enabled, which impacts performance significantly.

tasszz2k avatar Jun 03 '24 16:06 tasszz2k

if you don't want to change parameters of target redis you could use following code to restore above aof-preamble file

    public static void restore(File file, String targetUri) throws IOException, URISyntaxException {
        RedisURI turi = new RedisURI(targetUri);
        final ExampleClient target = new ExampleClient(turi.getHost(), turi.getPort());
        Configuration tconfig = Configuration.valueOf(turi);
        if (tconfig.getAuthPassword() != null) {
            Object auth = target.send(AUTH, tconfig.getAuthPassword().getBytes());
            System.out.println("AUTH:" + auth);
        }
        final AtomicInteger dbnum = new AtomicInteger(-1);
        Replicator r = dress(new RedisReplicator(file, FileType.MIXED, Configuration.defaultSetting()));

        r.addEventListener(new EventListener() {
            @Override
            public void onEvent(Replicator replicator, Event event) {
                
                // function since redis 7.0
                if (event instanceof DumpFunction) {
                    DumpFunction dfn = (DumpFunction) event;
                    Object r = target.restoreFunction(dfn.getSerialized(), true);
                    System.out.println(r);
                }
                
                // key value
                if (event instanceof DumpKeyValuePair) {
                    DumpKeyValuePair dkv = (DumpKeyValuePair) event;
                    // Step1: select db
                    DB db = dkv.getDb();
                    int index;
                    if (db != null && (index = (int) db.getDbNumber()) != dbnum.get()) {
                        target.send(SELECT, toByteArray(index));
                        dbnum.set(index);
                        System.out.println("SELECT:" + index);
                    }

                    // Step2: restore dump data
                    if (dkv.getExpiredMs() == null) {
                        Object r = target.restore(dkv.getKey(), 0L, dkv.getValue(), true);
                        System.out.println(r);
                    } else {
                        long ms = dkv.getExpiredMs() - System.currentTimeMillis();
                        if (ms <= 0) return;
                        Object r = target.restore(dkv.getKey(), ms, dkv.getValue(), true);
                        System.out.println(r);
                    }
                }

                // incremental commands
                if (event instanceof DefaultCommand) {
                    // Step3: sync aof command
                    DefaultCommand dc = (DefaultCommand) event;
                    Object r = target.send(dc.getCommand(), dc.getArgs());
                    System.out.println(r);
                }
            }
        });

        r.addCloseListener(new CloseListener() {
            @Override
            public void handle(Replicator replicator) {
                target.close();
            }
        });
        r.open();
    }

    public static Replicator dress(Replicator r) {
        r.setRdbVisitor(new DumpRdbVisitor(r));
        // ignore PING REPLCONF GETACK
        r.addCommandParser(CommandName.name("PING"), new PingParser());
        r.addCommandParser(CommandName.name("REPLCONF"), new ReplConfParser());
        //
        r.addCommandParser(CommandName.name("APPEND"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("SET"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("SETEX"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("MSET"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("DEL"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("SADD"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("HMSET"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("HSET"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("LSET"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("EXPIRE"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("EXPIREAT"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("GETSET"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("HSETNX"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("MSETNX"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("PSETEX"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("SETNX"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("SETRANGE"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("HDEL"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("LPOP"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("LPUSH"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("LPUSHX"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("LRem"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("RPOP"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("RPUSH"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("RPUSHX"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("ZREM"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("RENAME"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("INCR"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("DECR"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("INCRBY"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("DECRBY"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("PERSIST"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("SELECT"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("FLUSHALL"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("FLUSHDB"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("HINCRBY"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("ZINCRBY"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("MOVE"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("SMOVE"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("PFADD"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("PFCOUNT"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("PFMERGE"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("SDIFFSTORE"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("SINTERSTORE"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("SUNIONSTORE"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("ZADD"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("ZINTERSTORE"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("ZUNIONSTORE"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("BRPOPLPUSH"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("LINSERT"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("RENAMENX"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("RESTORE"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("PEXPIRE"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("PEXPIREAT"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("GEOADD"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("EVAL"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("EVALSHA"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("SCRIPT"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("PUBLISH"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("BITOP"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("BITFIELD"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("SETBIT"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("SREM"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("UNLINK"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("SWAPDB"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("MULTI"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("EXEC"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("ZREMRANGEBYSCORE"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("ZREMRANGEBYRANK"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("ZREMRANGEBYLEX"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("LTRIM"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("SORT"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("RPOPLPUSH"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("ZPOPMIN"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("ZPOPMAX"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("XACK"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("XADD"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("XCLAIM"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("XDEL"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("XGROUP"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("XTRIM"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("XSETID"), new DefaultCommandParser());
        // since redis 6.2
        r.addCommandParser(CommandName.name("COPY"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("LMOVE"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("BLMOVE"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("ZDIFFSTORE"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("GEOSEARCHSTORE"), new DefaultCommandParser());
        // since redis 7.0
        r.addCommandParser(CommandName.name("SPUBLISH"), new DefaultCommandParser());
        r.addCommandParser(CommandName.name("FUNCTION"), new DefaultCommandParser());
        return r;
    }

    /*
     * Jedis is not a reliable redis client.
     * For simplicity we use Jedis to show this example.
     * In production you need to replace following code to yours.
     */
    public static class ExampleClient implements Closeable {
    
        private Jedis jedis;

        public ExampleClient(final String host, final int port) {
            DefaultJedisClientConfig.Builder config = DefaultJedisClientConfig.builder();
            config.timeoutMillis(10000);
            this.jedis = new Jedis(new HostAndPort(host, port), config.build());
        }
    
        public Object send(Protocol.Command cmd, final byte[]... args) {
            Object r = jedis.sendCommand(cmd, args);
            if (r instanceof byte[]) {
                return Strings.toString(r);
            } else {
                return r;
            }
        }

        public Object send(final byte[] cmd, final byte[]... args) {
            return send(Protocol.Command.valueOf(Strings.toString(cmd).toUpperCase()), args);
        }

        public Object restore(byte[] key, long expired, byte[] dumped, boolean replace) {
            if (!replace) {
                return jedis.restore(key, expired, dumped);
            } else {
                return jedis.restore(key, expired, dumped, RestoreParams.restoreParams().replace());
            }
        }
        
        public Object restoreFunction(byte[] dumped, boolean replace) {
            if (!replace) {
                return jedis.functionRestore(dumped, FunctionRestorePolicy.APPEND);
            } else {
                return jedis.functionRestore(dumped, FunctionRestorePolicy.REPLACE);
            }
        }
    
        @Override
        public void close() {
            if (jedis != null) {
                jedis.close();
            }
        }
    }

restore(new File("/path/to/appendonly.aof"), "redis://target:6379")

leonchen83 avatar Jun 04 '24 02:06 leonchen83

more details refer to MigrationExample.java

leonchen83 avatar Jun 04 '24 02:06 leonchen83

it can directly generate rdb but you should very familiar rdb format, and you could use BaseRdbEncoder.java to generate rdb

leonchen83 avatar Jun 04 '24 02:06 leonchen83