Xml DB

View: New views

2 Messages — Rating Filter: Alert me

Xml DB

by Marc Deschamps :: Rate this Message:

Reply (Restricted by the Administrator) | Reply to Author | View Threaded | Show Only this Message

I'm in the process of evaluating Berkeley DBXML for an embedded xml data
store for the bank i work for (Société Générale in Paris).

The goal is to accelerate report production from massive xml source
(9gigs of XML growing fast).

To make this solution viable, i need to implement replication of dbxml
database over the network, and i can't find any example how to
implement replication in java. Application run on 2 J2EE servers.

Can you help me getting started? I would really appreciate.
I attached a java file of were i'm at actually if it can help getting me
started.

Thanks in advance.

Marc Deschamps
Loan Studio Project Manager
marc.deschamps@...

package core;

import com.sgcib.lso.dbxml.MyXMLDBEnv;
import com.sleepycat.dbxml.XmlContainer;
import com.sleepycat.dbxml.XmlDocument;
import com.sleepycat.dbxml.XmlDocumentConfig;
import com.sleepycat.dbxml.XmlManager;
import com.sleepycat.dbxml.XmlUpdateContext;
import java.util.LinkedList;
import org.jgroups.Address;
import org.jgroups.Channel;
import org.jgroups.JChannel;
import org.jgroups.MembershipListener;
import org.jgroups.Message;
import org.jgroups.MessageListener;
import org.jgroups.View;
import org.jgroups.blocks.PullPushAdapter;
import org.jgroups.util.Util;

/**
*
* @author Marc
*/
public class Sender implements Runnable
{

public Sender()
{
super();
}

public static void main(String[] args)
{
try
{
Sender main = new Sender();
main.run();
}
catch (Exception e)
{
System.err.println(e.getMessage());
e.printStackTrace(System.err);
}
}

public void run()
{
try
{
MyXMLDBEnv env = new MyXMLDBEnv("c:/www/data");
XmlManager mgr = env.getManager();
XmlContainer container = null;
try
{
container = mgr.openContainer("test.dbxml");
}
catch (Exception e)
{
container = mgr.createContainer("test.dbxml");
}
XmlDocumentConfig docConfig = XmlDocumentConfig.DEFAULT.setGenerateName(true);
XmlUpdateContext uCtx = mgr.createUpdateContext();
for(int i = 0; i < 100000; i++)
{
XmlDocument doc = mgr.createDocument();
doc.setContent("<test>test" + i + "</test>");
container.putDocument(doc, uCtx, docConfig);
}
System.out.println("sended 100000 documents");
while(true)
{
Thread.sleep(1000);
}
}
catch (Exception e)
{
System.err.println(e.getMessage());
e.printStackTrace(System.err);
}
return;
}
}

package core;

import com.sgcib.lso.dbxml.MyXMLDBEnv;
import com.sleepycat.dbxml.XmlContainer;
import com.sleepycat.dbxml.XmlDocument;
import com.sleepycat.dbxml.XmlDocumentConfig;
import com.sleepycat.dbxml.XmlManager;
import com.sleepycat.dbxml.XmlUpdateContext;
import java.util.LinkedList;
import org.jgroups.Address;
import org.jgroups.Channel;
import org.jgroups.JChannel;
import org.jgroups.MembershipListener;
import org.jgroups.Message;
import org.jgroups.MessageListener;
import org.jgroups.View;
import org.jgroups.blocks.PullPushAdapter;
import org.jgroups.util.Util;

/**
*
* @author Marc
*/
public class Receiver implements Runnable
{

public Receiver()
{
super();
}

public static void main(String[] args)
{
try
{
Receiver main = new Receiver();
main.run();
}
catch (Exception e)
{
System.err.println(e.getMessage());
e.printStackTrace(System.err);
}
}

public void run()
{
try
{
MyXMLDBEnv env = new MyXMLDBEnv("c:/www/data_replicated");
XmlManager mgr = env.getManager();
XmlContainer container = null;
try
{
container = mgr.openContainer("test.dbxml");
}
catch (Exception e)
{
container = mgr.createContainer("test.dbxml");
}
while(true)
{
Thread.sleep(1000);
}
}
catch (Exception e)
{
System.err.println(e.getMessage());
e.printStackTrace(System.err);
}
return;
}
}

package com.sgcib.lso.dbxml;

import com.sleepycat.db.DatabaseEntry;
import com.sleepycat.db.LogSequenceNumber;
import com.sleepycat.db.ReplicationTransport;
import com.sleepycat.db.internal.DbConstants;
import com.sleepycat.dbxml.XmlManagerConfig;
import java.io.File;

import com.sleepycat.db.DatabaseException;
import com.sleepycat.db.Environment;
import com.sleepycat.db.EnvironmentConfig;
import com.sleepycat.dbxml.XmlManager;
import java.io.Serializable;
import java.util.LinkedList;
import org.jgroups.Address;
import org.jgroups.Channel;
import org.jgroups.JChannel;
import org.jgroups.MembershipListener;
import org.jgroups.Message;
import org.jgroups.MessageListener;
import org.jgroups.View;
import org.jgroups.blocks.LockingException;
import org.jgroups.blocks.PullPushAdapter;
import org.jgroups.blocks.ReplicationManager;
import org.jgroups.blocks.ReplicationReceiver;
import org.jgroups.blocks.UpdateException;
import org.jgroups.blocks.Xid;
import org.jgroups.util.Util;

//Class used to open and close a Berkeley DB environment
public class MyXMLDBEnv implements ReplicationTransport, MessageListener, MembershipListener
{
// database
private Environment dbEnv = null;
private XmlManager mgr = null;
private boolean dbEnvIsOpen = false;
private File home = null;
private LinkedList history = new LinkedList();

//Replication
private Integer id;
private Integer masterId;

private Channel channel;
private PullPushAdapter adapter;
private final String group_name = "BerkeleyDBXMLGroup";
private final String props="UDP(mcast_addr=228.10.10.10;mcast_port=45566;bind_addr=127.0.0.1):"
+ "PING:"
+ "FD:"
+ "STABLE:"
+ "NAKACK:"
+ "UNICAST:"
+ "FRAG:"
+ "FLUSH:"
+ "GMS:"
+ "VIEW_ENFORCER:"
+ "STATE_TRANSFER:"
+ "QUEUE";

public MyXMLDBEnv(String path2DbEnv)
{
try
{
home = new File(path2DbEnv);
if (!home.isDirectory())
{
throw new Exception(home.getPath() + " does not exist or is not a directory.");
}

channel=new JChannel(props);
channel.setOpt(Channel.AUTO_RECONNECT, Boolean.TRUE);
channel.setOpt(Channel.AUTO_GETSTATE, Boolean.TRUE);
System.out.println("Connecting to " + group_name);
channel.connect(group_name);

adapter=new PullPushAdapter(channel, this, this);
channel.getState(null, 5000);
id = new Integer((int) channel.getView().getVid().getId());

EnvironmentConfig config = EnvironmentConfig.DEFAULT;
config.setCacheSize(50 * 1024 * 1024);
config.setAllowCreate(true);
config.setInitializeCache(true);
config.setTransactional(true);
config.setInitializeLocking(true);
config.setInitializeLogging(true);
config.setInitializeCache(true);
config.setInitializeReplication(true);
config.setErrorStream(System.err);
config.setMessageStream(System.out);
config.setReplicationTransport(id.intValue(), this);
dbEnv = new Environment(home, config);
dbEnv.checkpoint(null);
dbEnv.removeOldLogFiles();

XmlManagerConfig xmlConfig = new XmlManagerConfig();
xmlConfig.setAdoptEnvironment(true);
xmlConfig.setAllowAutoOpen(true);
xmlConfig.setAllowExternalAccess(true);

// Boolean used to know whether to close the environment
// when the cleanup() method is called.
this.dbEnvIsOpen = true;
this.mgr = new XmlManager(dbEnv, xmlConfig);
}
catch(Exception e)
{
e.printStackTrace(System.err);
throw new ExceptionInInitializerError(e.getMessage());
}
}

// Returns the path to the database environment
public File getDbEnvPath()
{
return home;
}

// Returns the database environment encapsulated by this class.
public Environment getEnvironment()
{
return dbEnv;
}

// Returns the XmlManager encapsulated by this class.
public XmlManager getManager()
{
return mgr;
}

// Used to close the environment
public void cleanup() throws DatabaseException
{
if (dbEnvIsOpen)
{
dbEnv.close();
dbEnvIsOpen = false;
}
}

public int send(Environment dbenv, DatabaseEntry control, DatabaseEntry rec, LogSequenceNumber lsn, int envid, boolean noBuffer, boolean permanent) throws DatabaseException
{
int result = 0;
try
{
MyMsg mymsg = new MyMsg(control, rec, envid);
Message msg = new Message(null, null, Util.objectToByteBuffer(mymsg));
channel.send(msg);
}
catch (Exception e)
{
result = -1;
}
return result;
}

public void receive(Message message)
{
try
{
Object o= Util.objectToByteBuffer(message.getObject());
if(o instanceof MyMsg)
{
MyMsg mymsg = (MyMsg) o;
dbEnv.processReplicationMessage(mymsg.control, mymsg.rec, mymsg.envid);
}
System.out.println(o + " [" + message.getSrc() + "]\n");
history.add(o);
}
catch(Exception e)
{
System.err.println("Chat.receive(): " + e);
}
return;
}

public byte[] getState()
{
try
{
return Util.objectToByteBuffer(history);
}
catch(Exception e)
{
return null;
}
}

public void setState(byte[] state)
{
try
{
history=(LinkedList)Util.objectFromByteBuffer(state);
}
catch(Exception e)
{
e.printStackTrace();
}
return;
}

public void viewAccepted(View new_view)
{
System.out.println("Received view " + new_view + '\n');
int members = channel.getView().getMembers().size();

if(members > 1)
{
System.out.println("We're now a group of " + members + " members!");
try
{
Thread t = new Thread()
{
public void run()
{
try
{
int i = dbEnv.electReplicationMaster(channel.getView().getMembers().size(), 2, 1, 1000);
masterId = new Integer(i);
}
catch (Exception e)
{
e.printStackTrace();
}
}
};

t.start();
t.join();

if(id.equals(masterId))
dbEnv.startReplication(new DatabaseEntry(), true);
else if(masterId.intValue() >= 0)
dbEnv.startReplication(new DatabaseEntry(), false);
}
catch (Exception e)
{
e.printStackTrace();
}
}
return;
}

public void suspect(Address suspected_mbr)
{
}

public void block()
{
}

class MyMsg implements Serializable
{

public MyMsg(DatabaseEntry control, DatabaseEntry rec, int envid)
{
this.control = control;
this.rec = rec;
this.envid = envid;
}
int envid;
DatabaseEntry control;
DatabaseEntry rec;
}

class MyElectionThread implements Runnable
{
public void run()
{
try
{
}
catch (Exception e)
{
throw new RuntimeException(e.getMessage());
}
}
}
}

No virus found in this outgoing message.
Checked by AVG Free Edition.
Version: 7.1.394 / Virus Database: 268.10.4/399 - Release Date: 25/07/2006

------------------------------------------
To remove yourself from this list, send an
email to xml-unsubscribe@...

Re: Xml DB

by George Feinberg-3 :: Rate this Message:

Reply (Restricted by the Administrator) | Reply to Author | View Threaded | Show Only this Message

Marc,

There are unfortunately no examples of Berkeley DB replication available
in Java or for BDB XML. There are examples
in C in the Berkeley DB tree. See db-4.3.29/examples_c/ex_repquote
and ex_rep.

There are several users who have done replication in BDB XML.
Maybe one of them will chime in.

Regards,

George

> I'm in the process of evaluating Berkeley DBXML for an embedded xml
> data store for the bank i work for (Société Générale in Paris).
>
> The goal is to accelerate report production from massive xml source
> (9gigs of XML growing fast).
>
> To make this solution viable, i need to implement replication of
> dbxml database over the network, and i can't find any example how
> to implement replication in java. Application run on 2 J2EE servers.
>
> Can you help me getting started? I would really appreciate.
> I attached a java file of were i'm at actually if it can help
> getting me started.
>
> Thanks in advance.
>
> Marc Deschamps
> Loan Studio Project Manager
> marc.deschamps@...
>
> package core;
>
> import com.sgcib.lso.dbxml.MyXMLDBEnv;
> import com.sleepycat.dbxml.XmlContainer;
> import com.sleepycat.dbxml.XmlDocument;
> import com.sleepycat.dbxml.XmlDocumentConfig;
> import com.sleepycat.dbxml.XmlManager;
> import com.sleepycat.dbxml.XmlUpdateContext;
> import java.util.LinkedList;
> import org.jgroups.Address;
> import org.jgroups.Channel;
> import org.jgroups.JChannel;
> import org.jgroups.MembershipListener;
> import org.jgroups.Message;
> import org.jgroups.MessageListener;
> import org.jgroups.View;
> import org.jgroups.blocks.PullPushAdapter;
> import org.jgroups.util.Util;
>
> /**
> *
> * @author Marc
> */
> public class Sender implements Runnable
> {
>
> public Sender()
> {
> super();
> }
>
> public static void main(String[] args)
> {
> try
> {
> Sender main = new Sender();
> main.run();
> }
> catch (Exception e)
> {
> System.err.println(e.getMessage());
> e.printStackTrace(System.err);
> }
> }
>
> public void run()
> {
> try
> {
> MyXMLDBEnv env = new MyXMLDBEnv("c:/www/data");
> XmlManager mgr = env.getManager();
> XmlContainer container = null;
> try
> {
> container = mgr.openContainer("test.dbxml");
> }
> catch (Exception e)
> {
> container = mgr.createContainer("test.dbxml");
> }
> XmlDocumentConfig docConfig =
> XmlDocumentConfig.DEFAULT.setGenerateName(true);
> XmlUpdateContext uCtx = mgr.createUpdateContext();
> for(int i = 0; i < 100000; i++)
> {
> XmlDocument doc = mgr.createDocument();
> doc.setContent("<test>test" + i + "</test>");
> container.putDocument(doc, uCtx, docConfig);
> }
> System.out.println("sended 100000 documents");
> while(true)
> {
> Thread.sleep(1000);
> }
> }
> catch (Exception e)
> {
> System.err.println(e.getMessage());
> e.printStackTrace(System.err);
> }
> return;
> }
> }
> package core;
>
> import com.sgcib.lso.dbxml.MyXMLDBEnv;
> import com.sleepycat.dbxml.XmlContainer;
> import com.sleepycat.dbxml.XmlDocument;
> import com.sleepycat.dbxml.XmlDocumentConfig;
> import com.sleepycat.dbxml.XmlManager;
> import com.sleepycat.dbxml.XmlUpdateContext;
> import java.util.LinkedList;
> import org.jgroups.Address;
> import org.jgroups.Channel;
> import org.jgroups.JChannel;
> import org.jgroups.MembershipListener;
> import org.jgroups.Message;
> import org.jgroups.MessageListener;
> import org.jgroups.View;
> import org.jgroups.blocks.PullPushAdapter;
> import org.jgroups.util.Util;
>
> /**
> *
> * @author Marc
> */
> public class Receiver implements Runnable
> {
>
> public Receiver()
> {
> super();
> }
>
> public static void main(String[] args)
> {
> try
> {
> Receiver main = new Receiver();
> main.run();
> }
> catch (Exception e)
> {
> System.err.println(e.getMessage());
> e.printStackTrace(System.err);
> }
> }
>
> public void run()
> {
> try
> {
> MyXMLDBEnv env = new MyXMLDBEnv("c:/www/data_replicated");
> XmlManager mgr = env.getManager();
> XmlContainer container = null;
> try
> {
> container = mgr.openContainer("test.dbxml");
> }
> catch (Exception e)
> {
> container = mgr.createContainer("test.dbxml");
> }
> while(true)
> {
> Thread.sleep(1000);
> }
> }
> catch (Exception e)
> {
> System.err.println(e.getMessage());
> e.printStackTrace(System.err);
> }
> return;
> }
> }
> package com.sgcib.lso.dbxml;
>
> import com.sleepycat.db.DatabaseEntry;
> import com.sleepycat.db.LogSequenceNumber;
> import com.sleepycat.db.ReplicationTransport;
> import com.sleepycat.db.internal.DbConstants;
> import com.sleepycat.dbxml.XmlManagerConfig;
> import java.io.File;
>
> import com.sleepycat.db.DatabaseException;
> import com.sleepycat.db.Environment;
> import com.sleepycat.db.EnvironmentConfig;
> import com.sleepycat.dbxml.XmlManager;
> import java.io.Serializable;
> import java.util.LinkedList;
> import org.jgroups.Address;
> import org.jgroups.Channel;
> import org.jgroups.JChannel;
> import org.jgroups.MembershipListener;
> import org.jgroups.Message;
> import org.jgroups.MessageListener;
> import org.jgroups.View;
> import org.jgroups.blocks.LockingException;
> import org.jgroups.blocks.PullPushAdapter;
> import org.jgroups.blocks.ReplicationManager;
> import org.jgroups.blocks.ReplicationReceiver;
> import org.jgroups.blocks.UpdateException;
> import org.jgroups.blocks.Xid;
> import org.jgroups.util.Util;
>
> //Class used to open and close a Berkeley DB environment
> public class MyXMLDBEnv implements ReplicationTransport,
> MessageListener, MembershipListener
> {
> // database
> private Environment dbEnv = null;
> private XmlManager mgr = null;
> private boolean dbEnvIsOpen = false;
> private File home = null;
> private LinkedList history = new LinkedList();
>
> //Replication
> private Integer id;
> private Integer masterId;
>
> private Channel channel;
> private PullPushAdapter adapter;
> private final String group_name = "BerkeleyDBXMLGroup";
> private final String props="UDP
> (mcast_addr=228.10.10.10;mcast_port=45566;bind_addr=127.0.0.1):"
> + "PING:"
> + "FD:"
> + "STABLE:"
> + "NAKACK:"
> + "UNICAST:"
> + "FRAG:"
> + "FLUSH:"
> + "GMS:"
> + "VIEW_ENFORCER:"
> + "STATE_TRANSFER:"
> + "QUEUE";
>
> public MyXMLDBEnv(String path2DbEnv)
> {
> try
> {
> home = new File(path2DbEnv);
> if (!home.isDirectory())
> {
> throw new Exception(home.getPath() + " does not
> exist or is not a directory.");
> }
>
> channel=new JChannel(props);
> channel.setOpt(Channel.AUTO_RECONNECT, Boolean.TRUE);
> channel.setOpt(Channel.AUTO_GETSTATE, Boolean.TRUE);
> System.out.println("Connecting to " + group_name);
> channel.connect(group_name);
>
> adapter=new PullPushAdapter(channel, this, this);
> channel.getState(null, 5000);
> id = new Integer((int) channel.getView().getVid().getId
> ());
>
> EnvironmentConfig config = EnvironmentConfig.DEFAULT;
> config.setCacheSize(50 * 1024 * 1024);
> config.setAllowCreate(true);
> config.setInitializeCache(true);
> config.setTransactional(true);
> config.setInitializeLocking(true);
> config.setInitializeLogging(true);
> config.setInitializeCache(true);
> config.setInitializeReplication(true);
> config.setErrorStream(System.err);
> config.setMessageStream(System.out);
> config.setReplicationTransport(id.intValue(), this);
> dbEnv = new Environment(home, config);
> dbEnv.checkpoint(null);
> dbEnv.removeOldLogFiles();
>
> XmlManagerConfig xmlConfig = new XmlManagerConfig();
> xmlConfig.setAdoptEnvironment(true);
> xmlConfig.setAllowAutoOpen(true);
> xmlConfig.setAllowExternalAccess(true);
>
> // Boolean used to know whether to close the environment
> // when the cleanup() method is called.
> this.dbEnvIsOpen = true;
> this.mgr = new XmlManager(dbEnv, xmlConfig);
> }
> catch(Exception e)
> {
> e.printStackTrace(System.err);
> throw new ExceptionInInitializerError(e.getMessage());
> }
> }
>
> // Returns the path to the database environment
> public File getDbEnvPath()
> {
> return home;
> }
>
> // Returns the database environment encapsulated by this class.
> public Environment getEnvironment()
> {
> return dbEnv;
> }
>
> // Returns the XmlManager encapsulated by this class.
> public XmlManager getManager()
> {
> return mgr;
> }
>
> // Used to close the environment
> public void cleanup() throws DatabaseException
> {
> if (dbEnvIsOpen)
> {
> dbEnv.close();
> dbEnvIsOpen = false;
> }
> }
>
> public int send(Environment dbenv, DatabaseEntry control,
> DatabaseEntry rec, LogSequenceNumber lsn, int envid, boolean
> noBuffer, boolean permanent) throws DatabaseException
> {
> int result = 0;
> try
> {
> MyMsg mymsg = new MyMsg(control, rec, envid);
> Message msg = new Message(null, null,
> Util.objectToByteBuffer(mymsg));
> channel.send(msg);
> }
> catch (Exception e)
> {
> result = -1;
> }
> return result;
> }
>
> public void receive(Message message)
> {
> try
> {
> Object o= Util.objectToByteBuffer(message.getObject());
> if(o instanceof MyMsg)
> {
> MyMsg mymsg = (MyMsg) o;
> dbEnv.processReplicationMessage(mymsg.control,
> mymsg.rec, mymsg.envid);
> }
> System.out.println(o + " [" + message.getSrc() + "]\n");
> history.add(o);
> }
> catch(Exception e)
> {
> System.err.println("Chat.receive(): " + e);
> }
> return;
> }
>
> public byte[] getState()
> {
> try
> {
> return Util.objectToByteBuffer(history);
> }
> catch(Exception e)
> {
> return null;
> }
> }
>
> public void setState(byte[] state)
> {
> try
> {
> history=(LinkedList)Util.objectFromByteBuffer(state);
> }
> catch(Exception e)
> {
> e.printStackTrace();
> }
> return;
> }
>
> public void viewAccepted(View new_view)
> {
> System.out.println("Received view " + new_view + '\n');
> int members = channel.getView().getMembers().size();
>
> if(members > 1)
> {
> System.out.println("We're now a group of " + members +
> " members!");
> try
> {
> Thread t = new Thread()
> {
> public void run()
> {
> try
> {
> int i = dbEnv.electReplicationMaster
> (channel.getView().getMembers().size(), 2, 1, 1000);
> masterId = new Integer(i);
> }
> catch (Exception e)
> {
> e.printStackTrace();
> }
> }
> };
>
> t.start();
> t.join();
>
> if(id.equals(masterId))
> dbEnv.startReplication(new DatabaseEntry(), true);
> else if(masterId.intValue() >= 0)
> dbEnv.startReplication(new DatabaseEntry(),
> false);
> }
> catch (Exception e)
> {
> e.printStackTrace();
> }
> }
> return;
> }
>
>
> public void suspect(Address suspected_mbr)
> {
> }
>
>
> public void block()
> {
> }
>
> class MyMsg implements Serializable
> {
>
> public MyMsg(DatabaseEntry control, DatabaseEntry rec, int
> envid)
> {
> this.control = control;
> this.rec = rec;
> this.envid = envid;
> }
> int envid;
> DatabaseEntry control;
> DatabaseEntry rec;
> }
>
> class MyElectionThread implements Runnable
> {
> public void run()
> {
> try
> {
> }
> catch (Exception e)
> {
> throw new RuntimeException(e.getMessage());
> }
> }
> }
> }

------------------------------------------
To remove yourself from this list, send an
email to xml-unsubscribe@...