S3特性测试

 

package amazons3;

import java.io.File;
import java.io.IOException;
import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.util.Date;
import java.util.Map;
import java.util.TreeMap;

import org.apache.commons.lang.StringUtils;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpHead;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.FileEntity;
import org.apache.http.entity.mime.MultipartEntity;
import org.apache.http.entity.mime.content.ContentBody;
import org.apache.http.entity.mime.content.FileBody;
import org.apache.http.entity.mime.content.StringBody;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.cookie.DateUtils;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;

/**
 * S3 client with apache httpclient 4.x
 * 
 * @author zhangpu
 * 
 */
public class S3HttpClient {

	private static final Logger logger = Logger.getLogger(S3HttpClient.class);

	static String accessKey = "AKIAJC2MNN5XZWLRB4UQ";
	static String secretKey = "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";
	static String bucket = "zhangpu";

	public static void main(String[] args) throws Exception {
		File localFile = new File("D:/temp/s3testfile.txt");
		String objectName = localFile.getName() + "1";
		// postObject(bucket, objectName, localFile, null, false, null, null);
		Map<String, String> protocols = new TreeMap<String, String>();
		Map<String, String> metas = new TreeMap<String, String>();
		metas.put("x-amz-meta-name", "zhangpu");
		metas.put("x-amz-meta-type", "type");
		putObject(bucket, objectName, localFile, "text/plant", false, null, metas);
		getObject(bucket, objectName, null, null, null, null);
		protocols.clear();
		protocols.put(HttpHeaders.IF_NONE_MATCH, "3e89d34c93742d0a92d4dce69a857e2f");
		protocols.put(HttpHeaders.CONTENT_ENCODING, "gzip");
		getObject(bucket, objectName, null, protocols, null, null);
		headObject(bucket, objectName, null, null);
		deleteObject(bucket, objectName, null);
	}

	/**
	 * POST方式上传文件
	 * 
	 * @param bucket
	 * @param objectName
	 * @param localFile
	 * @param contentType
	 * @param checkDigest
	 * @param protocols
	 * @param metas
	 * @return
	 * @throws Exception
	 */
	public static Header[] postObject(String bucket, String objectName, File localFile, String contentType, boolean checkDigest, Map<String, String> protocols,
			Map<String, String> metas) throws Exception {

		HttpClient httpClient = new DefaultHttpClient();
		if (StringUtils.isBlank(objectName)) {
			objectName = localFile.getName();
		}
		String url = "http://" + bucket + ".s3.amazonaws.com/" + objectName;
		String resource = "/" + bucket + "/" + objectName;
		HttpPost request = new HttpPost(url);

		Charset charset = Charset.forName("UTF-8");
		MultipartEntity mpEntity = new MultipartEntity();
		mpEntity.addPart("key", new StringBody("/user/${filename}", charset));
		if (StringUtils.isNotBlank(contentType)) {
			mpEntity.addPart("Content-Type", new StringBody(contentType, charset));
		}

		if (metas != null && metas.size() > 0) {
			for (Map.Entry<String, String> entry : metas.entrySet()) {
				mpEntity.addPart(entry.getKey(), new StringBody(entry.getValue(), charset));
			}
		}
		mpEntity.addPart("AWSAccessKeyId", new StringBody(accessKey, charset));
		String Signature = SignUtils.hmac(accessKey, secretKey, "POST", "", contentType, "", resource, protocols, metas);
		mpEntity.addPart("Signature", new StringBody(Signature, charset));

		ContentBody fileBody = new FileBody(localFile);
		mpEntity.addPart("file", fileBody);

		request.setEntity(mpEntity);
		printRequest(request);

		HttpResponse response = httpClient.execute(request);
		printResponse(response);
		HttpEntity entity = response.getEntity();
		if (entity != null && entity.getContentLength() > 0) {
			logger.info("entity:\n" + EntityUtils.toString(response.getEntity()));
		}
		logger.info("");
		logger.info("");

		return null;
	}

	/**
	 * 上传文件
	 * 
	 * @param bucket
	 * @param objectName
	 * @param localFile
	 * @param contentType
	 * @param md5
	 * @param protocols
	 * @param metas
	 */
	public static void putObject(String bucket, String objectName, File localFile, String contentType, boolean checkDigest, Map<String, String> protocols,
			Map<String, String> metas) throws Exception {

		HttpClient httpClient = new DefaultHttpClient();
		if (StringUtils.isBlank(objectName)) {
			objectName = localFile.getName();
		}
		String url = "http://" + bucket + ".s3.amazonaws.com/" + objectName;
		String resource = "/" + bucket + "/" + objectName;
		HttpPut request = new HttpPut(url);
		String contentMD5 = "";
		if (checkDigest) {
			contentMD5 = SignUtils.md5file(localFile);
		}
		Date date = new Date();
		String dateString = DateUtils.formatDate(date, DateUtils.PATTERN_RFC1036);
		String authorization = SignUtils.sign(accessKey, secretKey, "PUT", contentMD5, contentType, dateString, resource, protocols, metas);
		request.addHeader("Date", dateString);
		request.addHeader("Authorization", authorization);
		if (StringUtils.isNotBlank(contentMD5)) {
			request.addHeader("Content-MD5", contentMD5);
		}
		if (StringUtils.isNotBlank(contentType)) {
			request.addHeader("Content-Type", contentType);
		}
		marshallProtocol(request, protocols);
		marshallProtocol(request, metas);
		printRequest(request);
		FileEntity fileEntity = new FileEntity(localFile, contentType);
		request.setEntity(fileEntity);
		HttpResponse response = httpClient.execute(request);
		printResponse(response);
		HttpEntity entity = response.getEntity();
		if (entity != null && entity.getContentLength() > 0) {
			logger.info("entity:\n" + EntityUtils.toString(response.getEntity()));
		}
		logger.info("");
		logger.info("");
	}

	/**
	 * 下载文件
	 * 
	 * @param bucket
	 * @param objectName
	 * @param versionId
	 * @param protocols
	 * @param metas
	 * @throws IOException
	 */
	public static void getObject(String bucket, String objectName, String versionId, Map<String, String> protocols, Map<String, String> metas,
			Map<String, String> queries) throws IOException {
		HttpClient httpClient = new DefaultHttpClient();

		String query = marshallQuery(queries);
		String url = "http://" + bucket + ".s3.amazonaws.com/" + objectName + query;
		HttpGet request = new HttpGet(url);
		Date date = new Date();
		String dateString = DateUtils.formatDate(date, DateUtils.PATTERN_RFC1036);
		String resource = "/" + bucket + "/" + objectName + query;
		String authorization = SignUtils.sign(accessKey, secretKey, "GET", "", "", dateString, resource, protocols, metas);
		request.addHeader("Date", dateString);
		request.addHeader("Authorization", authorization);
		marshallProtocol(request, protocols);
		marshallProtocol(request, metas);

		printRequest(request);
		HttpResponse response = httpClient.execute(request);
		printResponse(response);
		HttpEntity entity = response.getEntity();
		if (entity != null && entity.getContentLength() > 0) {
			logger.info("entity:\n" + EntityUtils.toString(response.getEntity()));
		}
		logger.info("");
	}

	public static void headObject(String bucket, String objectName, String versionId, Map<String, String> protocols) throws IOException {

		HttpClient httpClient = new DefaultHttpClient();
		String url = "http://" + bucket + ".s3.amazonaws.com/" + objectName;

		HttpHead request = new HttpHead(url);
		Date date = new Date();
		String dateString = DateUtils.formatDate(date, DateUtils.PATTERN_RFC1036);
		String authorization = SignUtils.sign(accessKey, secretKey, "HEAD", "", "", dateString, "/" + bucket + "/" + objectName, null, null);
		request.addHeader("Date", dateString);
		request.addHeader("Authorization", authorization);
		marshallProtocol(request, protocols);

		printRequest(request);
		HttpResponse response = httpClient.execute(request);
		printResponse(response);

		HttpEntity entity = response.getEntity();
		if (entity != null && entity.getContentLength() > 0) {
			logger.info("entity:" + EntityUtils.toString(response.getEntity()));
		}
	}

	/**
	 * 删除对象
	 * 
	 * @param bucket
	 * @param objectName
	 * @param versionId
	 * @throws Exception
	 */
	public static void deleteObject(String bucket, String objectName, String versionId) throws Exception {
		HttpClient httpClient = new DefaultHttpClient();
		String url = "http://" + bucket + ".s3.amazonaws.com/" + URLEncoder.encode(objectName, "UTF-8");
		String resource = "/" + bucket + "/" + objectName;
		if (StringUtils.isNotBlank(versionId)) {
			url += "?versionId=" + versionId;
			resource += "?versionId=" + versionId;
		}
		HttpDelete request = new HttpDelete(url);
		Date date = new Date();
		String dateString = DateUtils.formatDate(date, DateUtils.PATTERN_RFC1036);
		String authorization = SignUtils.sign(accessKey, secretKey, "DELETE", "", "", dateString, resource, null, null);
		request.addHeader("Date", dateString);
		request.addHeader("Authorization", authorization);

		printRequest(request);
		HttpResponse response = httpClient.execute(request);
		printResponse(response);

		HttpEntity entity = response.getEntity();
		if (entity != null && entity.getContentLength() > 0) {
			logger.info("entity:" + EntityUtils.toString(response.getEntity()));
		}
	}

	/**
	 * 打印请求头
	 * 
	 * @param request
	 */
	private static void printRequest(HttpRequest request) {
		logger.info(request.getRequestLine());
		printHeader(request.getAllHeaders());
		logger.info("");
	}

	/**
	 * 打印响应头
	 * 
	 * @param response
	 */
	private static void printResponse(HttpResponse response) {
		logger.info(response.getStatusLine());
		printHeader(response.getAllHeaders());
	}

	/**
	 * 答应HTTP头
	 * 
	 * @param headers
	 */
	private static void printHeader(Header[] headers) {
		for (Header header : headers) {
			logger.info(header);
		}
	}

	/**
	 * 装配query
	 * 
	 * @param queries
	 * @return
	 */
	private static String marshallQuery(Map<String, String> queries) {
		String query = "";
		if (queries != null) {
			boolean first = true;
			for (Map.Entry<String, String> entry : queries.entrySet()) {
				query += (first ? "?" : "&") + entry.getKey() + "=" + entry.getValue();
				first = false;
			}
		}
		return query;
	}

	/**
	 * 设置请求协议
	 * 
	 * @param request
	 * @param protocols
	 */
	private static void marshallProtocol(HttpRequest request, Map<String, String> protocols) {
		if (protocols != null) {
			for (Map.Entry<String, String> entry : protocols.entrySet()) {
				request.addHeader(entry.getKey(), entry.getValue());
			}
		}
	}

}

 

 

package amazons3;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Map;

import javax.crypto.Mac;
import javax.crypto.spec.SecretKeySpec;

import org.apache.commons.codec.binary.Base64;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;

import sun.misc.BASE64Encoder;

public final class SignUtils {

	/**
	 * MD5文件
	 * 
	 * @param file
	 * @return
	 * @throws NoSuchAlgorithmException
	 * @throws IOException
	 */
	public static String md5file(File file) throws NoSuchAlgorithmException, IOException {
		MessageDigest messageDigest = MessageDigest.getInstance("MD5");
		BufferedInputStream in = new BufferedInputStream(new FileInputStream(file));
		byte[] buf = new byte[1024 * 100];
		int p = 0;
		while ((p = in.read(buf)) != -1) {
			messageDigest.update(buf, 0, p);
		}
		in.close();
		byte[] digest = messageDigest.digest();

		BASE64Encoder encoder = new BASE64Encoder();
		return encoder.encode(digest);
	}

	public static String simpleMd5File(File file) throws IOException {
		return DigestUtils.md5Hex(FileUtils.readFileToByteArray(file));
	}

	/**
	 * 计算签名
	 * 
	 * @param httpVerb
	 * @param contentMD5
	 * @param contentType
	 * @param date
	 * @param resource
	 * @param metas
	 * @return
	 */
	public static String sign(String accessKey, String secretKey, String httpVerb, String contentMD5, String contentType, String date, String resource,
			Map<String, String> protocols, Map<String, String> metas) {
		return "AWS" + " " + accessKey + ":" + hmac(accessKey, secretKey, httpVerb, contentMD5, contentType, date, resource, protocols, metas);
	}

	public static String hmac(String accessKey, String secretKey, String httpVerb, String contentMD5, String contentType, String date, String resource,
			Map<String, String> protocols, Map<String, String> metas) {

		String stringToSign = httpVerb + "\n" + StringUtils.trimToEmpty(contentMD5) + "\n" + StringUtils.trimToEmpty(contentType) + "\n" + StringUtils.trimToEmpty(date) + "\n";
		if (metas != null) {
			for (Map.Entry<String, String> entity : metas.entrySet()) {
				stringToSign += StringUtils.trimToEmpty(entity.getKey()) + ":" + StringUtils.trimToEmpty(entity.getValue()) + "\n";
			}
		}
		if (protocols != null) {
			for (Map.Entry<String, String> entity : protocols.entrySet()) {
				if (entity.getKey().startsWith("x-amz-")) {
					stringToSign += StringUtils.trimToEmpty(entity.getKey()) + ":" + StringUtils.trimToEmpty(entity.getValue()) + "\n";
				}
			}
		}
		stringToSign += resource;
		try {
			Mac mac = Mac.getInstance("HmacSHA1");
			byte[] keyBytes = secretKey.getBytes("UTF8");
			SecretKeySpec signingKey = new SecretKeySpec(keyBytes, "HmacSHA1");
			mac.init(signingKey);
			byte[] signBytes = mac.doFinal(stringToSign.getBytes("UTF8"));
			String signature = encodeBase64(signBytes);
			return signature;
		} catch (Exception e) {
			throw new RuntimeException("MAC CALC FAILED.");
		}

	}

	public static String encodeBase64(byte[] data) {
		String base64 = new String(Base64.encodeBase64(data));
		if (base64.endsWith("\r\n"))
			base64 = base64.substring(0, base64.length() - 2);
		if (base64.endsWith("\n"))
			base64 = base64.substring(0, base64.length() - 1);

		return base64;
	}
}