Revision 5920
Added by Matt Jones about 13 years ago
| src/edu/ucsb/nceas/metacat/dataone/CrudService.java | ||
|---|---|---|
| 30 | 30 |
import java.io.IOException; |
| 31 | 31 |
import java.io.InputStream; |
| 32 | 32 |
import java.io.OutputStream; |
| 33 |
import java.io.PrintWriter; |
|
| 34 |
import java.io.StringBufferInputStream; |
|
| 35 |
import java.security.MessageDigest; |
|
| 33 |
import java.security.NoSuchAlgorithmException; |
|
| 36 | 34 |
import java.sql.SQLException; |
| 37 |
import java.util.*; |
|
| 38 | 35 |
import java.text.DateFormat; |
| 39 | 36 |
import java.text.SimpleDateFormat; |
| 37 |
import java.util.Calendar; |
|
| 38 |
import java.util.Date; |
|
| 39 |
import java.util.Enumeration; |
|
| 40 |
import java.util.Hashtable; |
|
| 41 |
import java.util.List; |
|
| 42 |
import java.util.TimeZone; |
|
| 43 |
import java.util.Timer; |
|
| 44 |
import java.util.TimerTask; |
|
| 45 |
import java.util.Vector; |
|
| 40 | 46 |
|
| 41 |
import javax.servlet.ServletContext; |
|
| 42 | 47 |
import javax.servlet.http.HttpServletRequest; |
| 43 |
import javax.servlet.http.HttpServletResponse; |
|
| 44 | 48 |
|
| 45 | 49 |
import org.apache.commons.io.IOUtils; |
| 46 | 50 |
import org.apache.log4j.Logger; |
| 51 |
import org.dataone.service.exceptions.BaseException; |
|
| 47 | 52 |
import org.dataone.service.exceptions.IdentifierNotUnique; |
| 48 | 53 |
import org.dataone.service.exceptions.InsufficientResources; |
| 49 | 54 |
import org.dataone.service.exceptions.InvalidRequest; |
| ... | ... | |
| 55 | 60 |
import org.dataone.service.exceptions.ServiceFailure; |
| 56 | 61 |
import org.dataone.service.exceptions.UnsupportedType; |
| 57 | 62 |
import org.dataone.service.mn.MemberNodeCrud; |
| 58 |
import org.dataone.service.types.*; |
|
| 63 |
import org.dataone.service.types.AuthToken; |
|
| 64 |
import org.dataone.service.types.Checksum; |
|
| 65 |
import org.dataone.service.types.ChecksumAlgorithm; |
|
| 66 |
import org.dataone.service.types.DescribeResponse; |
|
| 67 |
import org.dataone.service.types.Event; |
|
| 68 |
import org.dataone.service.types.Identifier; |
|
| 69 |
import org.dataone.service.types.Log; |
|
| 70 |
import org.dataone.service.types.LogEntry; |
|
| 71 |
import org.dataone.service.types.NodeReference; |
|
| 72 |
import org.dataone.service.types.ObjectFormat; |
|
| 73 |
import org.dataone.service.types.ObjectList; |
|
| 74 |
import org.dataone.service.types.Principal; |
|
| 75 |
import org.dataone.service.types.SystemMetadata; |
|
| 76 |
import org.dataone.service.types.util.ServiceTypeUtil; |
|
| 59 | 77 |
import org.jibx.runtime.BindingDirectory; |
| 60 | 78 |
import org.jibx.runtime.IBindingFactory; |
| 61 | 79 |
import org.jibx.runtime.IMarshallingContext; |
| 62 | 80 |
import org.jibx.runtime.IUnmarshallingContext; |
| 63 | 81 |
import org.jibx.runtime.JiBXException; |
| 64 | 82 |
|
| 65 |
import org.dataone.service.types.Identifier; |
|
| 66 |
import org.dataone.service.types.util.ServiceTypeUtil; |
|
| 67 |
|
|
| 68 |
import com.gc.iotools.stream.is.InputStreamFromOutputStream; |
|
| 69 |
|
|
| 70 | 83 |
import edu.ucsb.nceas.metacat.AccessionNumber; |
| 71 | 84 |
import edu.ucsb.nceas.metacat.AccessionNumberException; |
| 72 |
import edu.ucsb.nceas.metacat.MetacatResultSet; |
|
| 73 |
import edu.ucsb.nceas.metacat.MetacatResultSet.Document; |
|
| 74 |
import edu.ucsb.nceas.metacat.DBQuery; |
|
| 75 | 85 |
import edu.ucsb.nceas.metacat.DocumentImpl; |
| 76 | 86 |
import edu.ucsb.nceas.metacat.EventLog; |
| 77 | 87 |
import edu.ucsb.nceas.metacat.IdentifierManager; |
| ... | ... | |
| 230 | 240 |
* documents in the object store. To work correctly, this should probably |
| 231 | 241 |
* be an adminstrative credential. |
| 232 | 242 |
*/ |
| 233 |
public void generateMissingSystemMetadata(AuthToken token) |
|
| 234 |
{
|
|
| 243 |
public void generateMissingSystemMetadata(AuthToken token) {
|
|
| 235 | 244 |
IdentifierManager im = IdentifierManager.getInstance(); |
| 236 | 245 |
//get the list of ids with no SM |
| 237 |
List<String> l = im.getLocalIdsWithNoSystemMetadata(); |
|
| 238 |
for(int i=0; i<l.size(); i++) |
|
| 239 |
{ //for each id, add a system metadata doc
|
|
| 240 |
String localId = l.get(i); |
|
| 241 |
System.out.println("Creating SystemMetadata for localId " + localId);
|
|
| 242 |
//get the document |
|
| 243 |
try |
|
| 244 |
{
|
|
| 245 |
//generate required system metadata fields from the document |
|
| 246 |
SystemMetadata sm = createSystemMetadata(localId, token); |
|
| 247 |
//insert the systemmetadata object |
|
| 248 |
SessionData sessionData = getSessionData(token); |
|
| 249 |
String smlocalid = insertSystemMetadata(sm, sessionData); |
|
| 250 |
System.out.println("setting access on SM doc with localid " + smlocalid);
|
|
| 251 |
handler.setAccess(metacatUrl, sessionData.getUserName(), smlocalid, |
|
| 252 |
"public", "4", "allow", "allowFirst"); |
|
| 253 |
|
|
| 254 |
String username = "public"; |
|
| 255 |
if(sessionData != null) |
|
| 256 |
{
|
|
| 257 |
username = sessionData.getUserName(); |
|
| 258 |
} |
|
| 259 |
EventLog.getInstance().log(metacatUrl, |
|
| 260 |
username, localId, "generateMissingSystemMetadata"); |
|
| 246 |
List<String> idList = im.getLocalIdsWithNoSystemMetadata(); |
|
| 247 |
for (String localId : idList) {
|
|
| 248 |
//for each id, add a system metadata doc |
|
| 249 |
generateMissingSystemMetadata(token, localId); |
|
| 250 |
} |
|
| 251 |
logCrud.info("generateMissingSystemMetadata(token)");
|
|
| 252 |
} |
|
| 253 |
|
|
| 254 |
/** |
|
| 255 |
* Generate SystemMetadata for a particular object with identifier localId. |
|
| 256 |
* This is a utility method for migration of existing objects |
|
| 257 |
* to DataONE where SystemMetadata is required for all objects. |
|
| 258 |
* |
|
| 259 |
* @param token an authtoken with appropriate permissions to read all |
|
| 260 |
* documents in the object store. To work correctly, this should |
|
| 261 |
* be an adminstrative credential. |
|
| 262 |
* @param localId the identifier of the object to be processed |
|
| 263 |
*/ |
|
| 264 |
public void generateMissingSystemMetadata(AuthToken token, String localId) {
|
|
| 265 |
System.out.println("Creating SystemMetadata for localId " + localId);
|
|
| 266 |
try {
|
|
| 267 |
//generate required system metadata fields from the document |
|
| 268 |
SystemMetadata sm = createSystemMetadata(localId, token); |
|
| 269 |
|
|
| 270 |
//insert the systemmetadata object |
|
| 271 |
SessionData sessionData = getSessionData(token); |
|
| 272 |
String smlocalid = insertSystemMetadata(sm, sessionData); |
|
| 273 |
System.out.println("setting access on SM doc with localid " + smlocalid);
|
|
| 274 |
handler.setAccess(metacatUrl, sessionData.getUserName(), smlocalid, "public", "4", "allow", "allowFirst"); |
|
| 275 |
|
|
| 276 |
String username = "public"; |
|
| 277 |
if (sessionData != null) {
|
|
| 278 |
username = sessionData.getUserName(); |
|
| 261 | 279 |
} |
| 262 |
catch(Exception e) |
|
| 263 |
{
|
|
| 264 |
e.printStackTrace(); |
|
| 265 |
System.out.println("Exception generating missing system metadata: " + e.getMessage());
|
|
| 266 |
logMetacat.error("Could not generate missing system metadata: " + e.getMessage());
|
|
| 267 |
} |
|
| 280 |
EventLog.getInstance().log(metacatUrl, username, localId, "generateMissingSystemMetadata"); |
|
| 281 |
} catch (Exception e) { // TODO: Please don't catch Exception -- it masks bad things
|
|
| 282 |
e.printStackTrace(); |
|
| 283 |
System.out.println("Exception generating missing system metadata: " + e.getMessage());
|
|
| 284 |
logMetacat.error("Could not generate missing system metadata: " + e.getMessage());
|
|
| 268 | 285 |
} |
| 269 |
logCrud.info("generateMissingSystemMetadata");
|
|
| 286 |
logCrud.info("generateMissingSystemMetadata(token, localId)");
|
|
| 270 | 287 |
} |
| 271 | 288 |
|
| 272 | 289 |
/** |
| ... | ... | |
| 1847 | 1864 |
* return an MD5 checksum for the stream |
| 1848 | 1865 |
* @param is |
| 1849 | 1866 |
* @return |
| 1867 |
* @throws IOException |
|
| 1868 |
* @throws NoSuchAlgorithmException |
|
| 1850 | 1869 |
*/ |
| 1851 |
public static String checksum(InputStream is) |
|
| 1852 |
throws Exception |
|
| 1870 |
public static String checksum(InputStream is) throws NoSuchAlgorithmException, IOException |
|
| 1853 | 1871 |
{
|
| 1854 | 1872 |
return checksum(is, "MD5"); |
| 1855 | 1873 |
} |
| 1856 | 1874 |
|
| 1857 | 1875 |
/** |
| 1858 | 1876 |
* produce a checksum for item using the given algorithm |
| 1877 |
* @throws IOException |
|
| 1878 |
* @throws NoSuchAlgorithmException |
|
| 1859 | 1879 |
*/ |
| 1860 |
public static String checksum(InputStream is, String algorithm) |
|
| 1861 |
throws Exception |
|
| 1880 |
public static String checksum(InputStream is, String algorithm) throws NoSuchAlgorithmException, IOException |
|
| 1862 | 1881 |
{
|
| 1863 | 1882 |
return ServiceTypeUtil.checksum(is, ChecksumAlgorithm.convert(algorithm)).getValue(); |
| 1864 | 1883 |
} |
| ... | ... | |
| 1904 | 1923 |
|
| 1905 | 1924 |
/** |
| 1906 | 1925 |
* create system metadata with a specified id, doc and format |
| 1926 |
* @throws McdbDocNotFoundException |
|
| 1927 |
* @throws SQLException |
|
| 1928 |
* @throws AccessionNumberException |
|
| 1929 |
* @throws NumberFormatException |
|
| 1930 |
* @throws IOException |
|
| 1931 |
* @throws NoSuchAlgorithmException |
|
| 1932 |
* @throws PropertyNotFoundException |
|
| 1907 | 1933 |
*/ |
| 1908 |
private SystemMetadata createSystemMetadata(String localId, AuthToken token) |
|
| 1909 |
throws Exception |
|
| 1910 |
{
|
|
| 1934 |
private SystemMetadata createSystemMetadata(String localId, AuthToken token) |
|
| 1935 |
throws McdbDocNotFoundException, NumberFormatException, AccessionNumberException, |
|
| 1936 |
SQLException, NoSuchAlgorithmException, IOException, PropertyNotFoundException, BaseException {
|
|
| 1937 |
|
|
| 1911 | 1938 |
IdentifierManager im = IdentifierManager.getInstance(); |
| 1912 | 1939 |
Hashtable<String, Object> docInfo = im.getDocumentInfo(localId); |
| 1913 |
|
|
| 1940 |
|
|
| 1914 | 1941 |
//get the document text |
| 1915 | 1942 |
int rev = im.getLatestRevForLocalId(localId); |
| 1916 | 1943 |
Identifier identifier = new Identifier(); |
| 1917 |
try |
|
| 1918 |
{
|
|
| 1944 |
try {
|
|
| 1919 | 1945 |
identifier.setValue(im.getGUID(localId, rev)); |
| 1920 |
} |
|
| 1921 |
catch(McdbDocNotFoundException mcdbe) |
|
| 1922 |
{ //we're creating a new SM doc for a doc that is not in the identifier table
|
|
| 1923 |
//so we need to add it to |
|
| 1946 |
} catch (McdbDocNotFoundException mcdbe) {
|
|
| 1947 |
//we're creating a new SM doc for a doc that is not in the identifier table |
|
| 1948 |
//so we need to add it to |
|
| 1924 | 1949 |
System.out.println("No guid in the identifier table. adding it for " + localId);
|
| 1925 | 1950 |
im.createMapping(localId, localId); |
| 1926 | 1951 |
System.out.println("mapping created for " + localId);
|
| 1927 | 1952 |
AccessionNumber accNum = new AccessionNumber(localId, "NONE"); |
| 1928 | 1953 |
identifier.setValue(im.getGUID(accNum.getDocid(), rev)); |
| 1929 | 1954 |
} |
| 1930 |
|
|
| 1955 |
|
|
| 1931 | 1956 |
System.out.println("creating system metadata for guid " + identifier.getValue());
|
| 1932 | 1957 |
InputStream is = this.get(token, identifier); |
| 1958 |
SystemMetadata sm = new SystemMetadata(); |
|
| 1933 | 1959 |
|
| 1934 |
SystemMetadata sm = new SystemMetadata(); |
|
| 1935 | 1960 |
//set the id |
| 1936 | 1961 |
sm.setIdentifier(identifier); |
| 1937 |
|
|
| 1962 |
|
|
| 1938 | 1963 |
//set the object format |
| 1939 |
String doctype = (String)docInfo.get("doctype");
|
|
| 1940 |
ObjectFormat format = ObjectFormat.convert((String)docInfo.get("doctype"));
|
|
| 1941 |
if(format == null) |
|
| 1942 |
{
|
|
| 1943 |
if(doctype.trim().equals("BIN"))
|
|
| 1944 |
{
|
|
| 1964 |
String doctype = (String) docInfo.get("doctype");
|
|
| 1965 |
ObjectFormat format = ObjectFormat.convert((String) docInfo.get("doctype"));
|
|
| 1966 |
if (format == null) {
|
|
| 1967 |
if (doctype.trim().equals("BIN")) {
|
|
| 1945 | 1968 |
format = ObjectFormat.OCTET_STREAM; |
| 1946 |
} |
|
| 1947 |
else |
|
| 1948 |
{
|
|
| 1969 |
} else {
|
|
| 1949 | 1970 |
format = ObjectFormat.convert("text/plain");
|
| 1950 | 1971 |
} |
| 1951 | 1972 |
} |
| 1952 | 1973 |
sm.setObjectFormat(format); |
| 1953 |
|
|
| 1974 |
|
|
| 1954 | 1975 |
//create the checksum |
| 1955 | 1976 |
String checksumS = checksum(is); |
| 1956 | 1977 |
ChecksumAlgorithm ca = ChecksumAlgorithm.convert("MD5");
|
| ... | ... | |
| 1958 | 1979 |
checksum.setValue(checksumS); |
| 1959 | 1980 |
checksum.setAlgorithm(ca); |
| 1960 | 1981 |
sm.setChecksum(checksum); |
| 1961 |
|
|
| 1982 |
|
|
| 1962 | 1983 |
//set the size |
| 1963 | 1984 |
is = this.get(token, identifier); |
| 1964 | 1985 |
sm.setSize(sizeOfStream(is)); |
| 1965 |
|
|
| 1986 |
|
|
| 1966 | 1987 |
//submitter |
| 1967 | 1988 |
Principal p = new Principal(); |
| 1968 |
p.setValue((String)docInfo.get("user_owner"));
|
|
| 1989 |
p.setValue((String) docInfo.get("user_owner"));
|
|
| 1969 | 1990 |
sm.setSubmitter(p); |
| 1970 | 1991 |
sm.setRightsHolder(p); |
| 1971 |
try |
|
| 1972 |
{
|
|
| 1973 |
Date dateCreated = parseMetacatDate((String)docInfo.get("date_created"));
|
|
| 1992 |
try {
|
|
| 1993 |
Date dateCreated = parseMetacatDate((String) docInfo.get("date_created"));
|
|
| 1974 | 1994 |
sm.setDateUploaded(dateCreated); |
| 1975 |
Date dateUpdated = parseMetacatDate((String)docInfo.get("date_updated"));
|
|
| 1995 |
Date dateUpdated = parseMetacatDate((String) docInfo.get("date_updated"));
|
|
| 1976 | 1996 |
sm.setDateSysMetadataModified(dateUpdated); |
| 1977 |
} |
|
| 1978 |
catch(Exception e) |
|
| 1979 |
{
|
|
| 1997 |
} catch (Exception e) {
|
|
| 1980 | 1998 |
System.out.println("POSSIBLE ERROR: couldn't parse a date: " + e.getMessage());
|
| 1981 | 1999 |
Date dateCreated = new Date(); |
| 1982 | 2000 |
sm.setDateUploaded(dateCreated); |
| ... | ... | |
| 1984 | 2002 |
sm.setDateSysMetadataModified(dateUpdated); |
| 1985 | 2003 |
} |
| 1986 | 2004 |
NodeReference nr = new NodeReference(); |
| 1987 |
//TODO: this should be set to be something more meaningful once the registry is up |
|
| 1988 |
nr.setValue("metacat");
|
|
| 2005 |
nr.setValue(PropertyService.getProperty("dataone.memberNodeId"));
|
|
| 1989 | 2006 |
sm.setOriginMemberNode(nr); |
| 1990 | 2007 |
sm.setAuthoritativeMemberNode(nr); |
| 2008 |
|
|
| 2009 |
// TODO: Need to set describes/describedBy |
|
| 2010 |
|
|
| 1991 | 2011 |
return sm; |
| 1992 | 2012 |
} |
| 1993 | 2013 |
|
Also available in: Unified diff
Updated to most recent DataONE libraries. Updated CrudService to set the correct origin MN and auth MN in system metadata. Refactored exception passing. More work to come in generating SystemMetadata.