Project

General

Profile

« Previous | Next » 

Revision 5636

Added by berkley about 14 years ago

getting closer to a non-memory bound solution for mime multipart de/encoding

View differences:

test/edu/ucsb/nceas/metacat/restservice/ResourceHandlerTest.java
1
/**
2
 *  '$RCSfile$'
3
 *  Copyright: 2010 Regents of the University of California and the
4
 *              National Center for Ecological Analysis and Synthesis
5
 *  Purpose: To test the Access Controls in metacat by JUnit
6
 *
7
 *   '$Author: berkley $'
8
 *     '$Date: 2010-05-03 14:26:08 -0700 (Fri, 14 Aug 2009) $'
9
 * '$Revision: 5027 $'
10
 *
11
 * This program is free software; you can redistribute it and/or modify
12
 * it under the terms of the GNU General Public License as published by
13
 * the Free Software Foundation; either version 2 of the License, or
14
 * (at your option) any later version.
15
 *
16
 * This program is distributed in the hope that it will be useful,
17
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19
 * GNU General Public License for more details.
20
 *
21
 * You should have received a copy of the GNU General Public License
22
 * along with this program; if not, write to the Free Software
23
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
24
 */
25

  
26
package edu.ucsb.nceas.metacat.restservice;
27

  
28
import java.util.*;
29
import java.io.*;
30

  
31
import edu.ucsb.nceas.MCTestCase;
32
import edu.ucsb.nceas.metacat.dataone.CrudServiceTest;
33

  
34
import junit.framework.Test;
35
import junit.framework.TestSuite;
36

  
37
import org.apache.commons.io.IOUtils;
38

  
39
/**
40
 * @author berkley
41
 * class to test ResourceHandler
42
 */
43
public class ResourceHandlerTest extends MCTestCase
44
{
45
    public ResourceHandlerTest(String name)
46
    {
47
        super(name);
48
    }
49

  
50
    /**
51
     * Create a suite of tests to be run together
52
     */
53
    public static Test suite() 
54
    {
55
        TestSuite suite = new TestSuite();
56
        
57
        suite.addTest(new ResourceHandlerTest("testProcessMMP"));
58
        
59
        return suite;
60
    }
61
    
62
    public void testProcessMMP()
63
    {
64
        try
65
        {
66
            File f = new File("test/testMimeInput.txt");
67
            FileInputStream fis = new FileInputStream(f);
68
            ResourceHandler rh = new ResourceHandler(null, null, null);
69
            //Hashtable h = rh.processMMP(fis);
70
            InputStream is = rh.processMMP(fis);
71
            String s = IOUtils.toString(is);
72
            System.out.println("Final Stream: " + s);
73
        }
74
        catch(Exception e)
75
        {
76
            fail("Unexpected error in testProcessMMP: " + e.getMessage());
77
        }
78
        
79
    }
80
}
src/edu/ucsb/nceas/metacat/restservice/ResourceHandler.java
69 69
import edu.ucsb.nceas.metacat.util.RequestUtil;
70 70
import edu.ucsb.nceas.metacat.util.SessionData;
71 71
import org.dataone.service.streaming.util.StreamUtil;
72

  
73
import com.gc.iotools.stream.is.InputStreamFromOutputStream;
72 74
/**
73 75
 * 
74 76
 * Implements Earthgrid REST API to Metacat <br/><br/> 
......
1012 1014
     * @param is
1013 1015
     * @return
1014 1016
     */
1015
    private Hashtable processMMP(InputStream is)
1017
    /*private Hashtable processMMP(InputStream is)
1016 1018
      throws IOException
1017 1019
    {
1018 1020
        //TODO: verify that this is how the RFC for MMP should work
......
1022 1024
        InputStream object = null;
1023 1025
        InputStream sysmeta = null;
1024 1026
        String s = IOUtils.toString(is);
1025
        //System.out.println("mime: " + s);
1027
        System.out.println("mime: " + s);
1026 1028
        //figure out what the boundary marker is
1027 1029
        String searchString = "boundary=";
1028 1030
        int searchStringIndex = s.indexOf(searchString);
......
1055 1057
        //System.out.println("sm: \"" + sm + "\"");
1056 1058
        return h;
1057 1059
    }
1058
    
1059
    /*private Hashtable processMMP(InputStream is)
1060
    */
1061
    protected InputStream processMMP(final InputStream is)
1060 1062
      throws IOException
1061 1063
    {
1062
        String[] searchStrings = {"boundary=", "Content-Disposition: attachment; filename=systemmetadata",
1064
        Hashtable<String, InputStream> partsHash = new Hashtable<String, InputStream>();
1065
        final InputStreamFromOutputStream<String> objectStream = 
1066
            new InputStreamFromOutputStream<String>() 
1067
        {
1068
            @Override
1069
            public String produce(final OutputStream dataSink) throws Exception 
1070
            {
1071
                System.out.println("producing data");
1072
                String[] searchStrings = {"boundary=", "Content-Disposition: attachment; filename=systemmetadata",
1063 1073
                "Content-Disposition: attachment; filename=object"};
1064
        byte[] b = new byte[1024];
1065
        int numread = is.read(b, 0, 1024);
1066
        int ssi = 0;
1067
        while(numread != -1)
1068
        {
1069
            String s = new String(b, 0, numread);
1070
            int[] result = StreamUtil.lookForMatch(searchStrings[ssi], s);
1071
            
1072
        }
1073
    }*/
1074
                String boundary = "";
1075
                String searchSegment = null;
1076
                byte[] b = new byte[1024];
1077
                int numread = is.read(b, 0, 1024);
1078
                String s = new String(b, 0, numread);
1079
                int ssi = 0;
1080
                boolean doneWithCurrentArray = false;
1081
                boolean searchForBoundary = false;
1082
                int currentArrayIndex = 0;
1083
                
1084
                while(numread != -1)
1085
                {
1086
                    String searchString;
1087
                    if(!searchForBoundary)
1088
                    {
1089
                        if(ssi == searchStrings.length)
1090
                        {
1091
                            break;
1092
                        }
1093
                        searchString = searchStrings[ssi];
1094
                    }
1095
                    else
1096
                    {
1097
                        searchString = boundary;
1098
                    }
1099
                    
1100
                    System.out.println("searchString: " + searchString);
1101
                    System.out.println("done with array: " + doneWithCurrentArray);
1102
                    System.out.println("searchForBoundary: " + searchForBoundary);
1103
                    if(doneWithCurrentArray)
1104
                    {
1105
                        System.out.println("Getting new chunk");
1106
                        numread = is.read(b, 0, 1024);
1107
                        s = new String(b, 0, numread);
1108
                    }
1109
                    
1110
                    System.out.println("/////////////////current chunk (s): \n" + s + "\n////////////end current chunk");
1111
                    
1112
                    int[] result = StreamUtil.lookForMatch(searchString, s);
1113
                    System.out.println("got result from StreamUtil: " + result[0] + ", " + result[1]);
1114
                    if(result[0] != -1 && result[1] == searchString.length())
1115
                    { //we have the whole search string in this group of bytes
1116
                        System.out.println("1");
1117
                        if(searchForBoundary)
1118
                        {
1119
                            System.out.println("2");
1120
                            //find the boundary, chop it off the end and inc ssi
1121
                            String strToWrite = s.substring(0, s.indexOf(boundary, currentArrayIndex));
1122
                            System.out.println("XXXX 1 writing string: " + strToWrite + "\nXXXX");
1123
                            dataSink.write(strToWrite.getBytes());
1124
                            doneWithCurrentArray = false; 
1125
                            currentArrayIndex = s.indexOf(boundary, currentArrayIndex) + 1;
1126
                            searchForBoundary = false;
1127
                            ssi++;
1128
                        }
1129
                        else if(ssi == 0)
1130
                        { //we're looking for the boundary marker
1131
                            int searchStringIndex = s.indexOf(searchString);
1132
                            if(s.indexOf("\"", searchStringIndex + searchString.length() + 1) == -1)
1133
                            { //the end of the boundary is in the next byte array
1134
                                boundary = s.substring(searchStringIndex + searchString.length() + 1, s.length());
1135
                                doneWithCurrentArray = true;
1136
                            }
1137
                            else if(!boundary.startsWith("--"))
1138
                            { //we can read the whole boundary from this byte array
1139
                                boundary = s.substring(searchStringIndex + searchString.length() + 1, 
1140
                                    s.indexOf("\"", searchStringIndex + searchString.length() + 1));
1141
                                boundary = "--" + boundary;
1142
                                ssi++;
1143
                                doneWithCurrentArray = false;
1144
                            }
1145
                            else
1146
                            { //we're now reading the 2nd byte array to get the rest of the boundary
1147
                                searchString = "\"";
1148
                                searchStringIndex = s.indexOf(searchString);
1149
                                boundary += s.substring(0, searchStringIndex);
1150
                                boundary = "--" + boundary;
1151
                                ssi++;
1152
                                doneWithCurrentArray = false;
1153
                            }
1154
                            System.out.println("boundary: " + boundary);
1155
                        }
1156
                        else if(ssi == 1 || ssi == 2)
1157
                        { 
1158
                            int searchStringIndex = s.indexOf(searchString);
1159
                            //find the beginning of the system metadata or data object and 
1160
                            //start writing it to the dataSink
1161
                            int smBeginIndex = s.indexOf(searchString) + searchString.length() + 2;
1162
                            searchForBoundary = true;
1163
                            int boundaryIndex = s.indexOf(boundary, smBeginIndex);
1164
                            int end = s.length();
1165
                            if(boundaryIndex != -1)
1166
                            {
1167
                                end = boundaryIndex;
1168
                                ssi++;
1169
                                searchForBoundary = false;
1170
                            }
1171
                            currentArrayIndex = end + 1;
1172
                            String strToWrite = s.substring(smBeginIndex, end);
1173
                            System.out.println("XXXX 2 writing string: " + strToWrite + "\nXXXX");
1174
                            dataSink.write(strToWrite.getBytes());
1175
                            if(end == s.length())
1176
                            {
1177
                                System.out.println("DONE WITH ARRAY");
1178
                                doneWithCurrentArray = true;
1179
                                currentArrayIndex = 0;
1180
                            }
1181
                            else
1182
                            {
1183
                                System.out.println("NOT DONE WITH ARRAY");
1184
                                doneWithCurrentArray = false;
1185
                            }
1186
                        }
1187
                    }
1188
                    else if(result[1] != searchString.length() && result[0] != 0)
1189
                    {  //part of the search string is on the end of this byte group
1190
                        doneWithCurrentArray = true;
1191
                        searchForBoundary = false;
1192
                        searchSegment = s.substring(result[0], s.length());
1193
                        System.out.println("searchSegment1: " + searchSegment);
1194
                    }
1195
                    else if(result[1] != searchString.length() && result[0] == 0)
1196
                    {  //we have the end of the searchString at the beginning of this byte group
1197
                        if(searchSegment != null)
1198
                        {
1199
                            searchSegment += s.substring(0, result[1]);
1200
                            System.out.println("searchSegment2: " + searchSegment);
1201
                            searchForBoundary = true;
1202
                            doneWithCurrentArray = true;
1203
                            ssi++;
1204
                            String strToWrite = s.substring(result[1] + 1, s.length());
1205
                            System.out.println("XXXX 3 writing string: " + strToWrite + "\nXXXX");
1206
                            dataSink.write(strToWrite.getBytes());
1207
                        }
1208
                    }
1209
                    else
1210
                    {
1211
                        if(searchForBoundary)
1212
                        {
1213
                            //we're in between the start and end of a section 
1214
                            //so just write this to the stream
1215
                            System.out.println("XXXX 4 writing string: " + s + "\nXXXX");
1216
                            dataSink.write(s.getBytes());
1217
                        }
1218
                    }
1219
                }
1220
                dataSink.flush();
1221
                return "Completed";
1222
            }
1223
        };
1224
        
1225
        //return partsHash;
1226
        return objectStream;
1227
    }
1074 1228
    
1075 1229
    /**
1076 1230
     * Earthgrid API > Put Service >Put Function : calls MetacatHandler > handleInsertOrUpdateAction 
......
1095 1249
            InputStream object = null;
1096 1250
            InputStream sysmeta = null;
1097 1251
            
1098
            Hashtable parts = processMMP(request.getInputStream());
1252
            Hashtable parts/* = processMMP(request.getInputStream());*/ = new Hashtable();
1099 1253
            object = (InputStream)parts.get("object");
1100 1254
            
1101 1255
            sysmeta = (InputStream)parts.get("systemmetadata");
......
1196 1350
        } /*catch (MessagingException e) {
1197 1351
            ServiceFailure sf = new ServiceFailure("1000", e.getMessage());
1198 1352
            serializeException(sf, out);
1199
        }*/ catch (IOException e) {
1353
        } catch (IOException e) {
1200 1354
            response.setStatus(500);
1201 1355
            ServiceFailure sf = new ServiceFailure("1000", e.getMessage());
1202 1356
            serializeException(sf, out);
1203
        } catch (JiBXException e) {
1357
        }*/ catch (JiBXException e) {
1204 1358
            response.setStatus(500);
1205 1359
            e.printStackTrace(System.out);
1206 1360
            InvalidSystemMetadata ism = new InvalidSystemMetadata("1080", e.getMessage());

Also available in: Unified diff