【玩转大数据】2.java操作HDFS的API

下面记录通过java操作hdfs的API流程

添加pom依赖

1
2
3
4
5
6
<!--添加hadoop依赖,版本最好与服务器的hadoop版本一致-->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.9.2</version>
</dependency>

测试类

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
package com.imooc.hadoop.hadooptrain;

import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.*;
import java.net.URI;

/**
* Hadoop HDFS JAVA api操作
*/
@Data
@Slf4j
public class HDFSApp {

public static final String HDFS_PATH = "hdfs://192.168.2.127:8020";

FileSystem fileSystem = null;
Configuration configuration = null;

/**
* 创建HDFS目录
* @throws Exception
*/
@Test
public void mkdir() throws Exception {
fileSystem.mkdirs(new Path("/hdfsapi/test"));
}

/**
* 创建文件
*/
@Test
public void create() throws Exception {
FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/hdfsapi/test/a.txt"));
fsDataOutputStream.write("hello hadoop".getBytes());
fsDataOutputStream.flush();
fsDataOutputStream.close();
}

/**
* 查看HDFS文件的内容
*/
@Test
public void cat() throws Exception {
FSDataInputStream in = fileSystem.open(new Path("/hdfsapi/test/a.txt"));
IOUtils.copyBytes(in, System.out, 1024);
in.close();

}


/**
* 重命名
*/
@Test
public void rename() throws IOException {
Path oldPath = new Path("/hdfsapi/test/a.txt");
Path newPath = new Path("/hdfsapi/test/b.txt");
fileSystem.rename(oldPath, newPath);
}

/**
* 文件上传到HDFS
*/
@Test
public void copyFromLocalFile() throws Exception{
Path localPath = new Path("/Users/hdw/Desktop/land.zip");
Path hdfsPath = new Path("/hdfsapi/test");
fileSystem.copyFromLocalFile(localPath, hdfsPath);
}

/**
* 文件上传到HDFS(进度条版)
*/
@Test
public void copyFromLocalFileWithProgress() throws Exception{
Path localPath = new Path("/Users/hdw/Desktop/Transform/jdk-8u181-linux-x64.tar.gz");
Path hdfsPath = new Path("/hdfsapi/test");
fileSystem.copyFromLocalFile(localPath, hdfsPath);

InputStream in = new BufferedInputStream(
new FileInputStream(
new File("/Users/hdw/Desktop/Transform/jdk-8u181-linux-x64.tar.gz")));
FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/hdfsapi/test/jdk-8u181-linux-x64.tar.gz"), new Progressable() {
@Override
public void progress() {
System.out.print(".");//带进度提醒信息
}
});

IOUtils.copyBytes(in,fsDataOutputStream,4096);
}

/**
* 下载文件到本地
*/
@Test
public void copyToLocalFile()throws Exception {
Path localPath = new Path("/Users/hdw/Desktop");
Path hdfsPath = new Path("/hdfsapi/test/jdk-8u181-linux-x64.tar.gz");
fileSystem.copyToLocalFile(hdfsPath,localPath);
}

/**
* 查看某个目录下的所有文件
*/
@Test
public void listFiles() throws Exception{
FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/"));
for (FileStatus fileStatus : fileStatuses) {
String isDir = fileStatus.isDirectory() ? "文件夹" : "文件";
short replication = fileStatus.getReplication();//副本
long len = fileStatus.getLen();//文件大小
String path = fileStatus.getPath().toString();
System.out.println("isDir:" + isDir + "\t" + "replication:" + replication + "\t" + "len:" + len + "\t\t" + "path" + path);
}
}


/**
* 删除
*/
@Test
public void delete() throws Exception {
fileSystem.delete(new Path("/hdfsapi/test/"), true);//递归删除
}



@Before
public void setUp() throws Exception {
System.out.println("HDFSApp.setUp");
configuration = new Configuration();
fileSystem = FileSystem.get(new URI(HDFS_PATH),configuration,"root");
}

@After
public void tearDown()throws Exception {
configuration = null;
fileSystem = null;
System.out.println("HDFSApp.tearDown");
}

}
-------------本文结束感谢您的阅读😜-------------
喜欢较真儿的平头哥 wechat
微信扫一扫,没事常联系 [请注明博客添加]
支持原创,支持开源,感谢打赏,不懈努力。
点击打赏,可领取支付宝红包,不信你试试😏😏😏!
0%