hadoop学习笔记 day02 hadoop Java环境的API

package day1; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.hadoop.fs.FileSystem; import org.junit.Before; import org.junit.Test; import java.io.*; import java.util.HashMap; import java.util.Map; public class HDFSdemo { FileSystem fs = null; //客户端Java的API //获得客户端对象 @Before public void init() throws IOException { System.setProperty("HADOOP_USER_NAME","root"); Configuration conf = new Configuration(); conf.set("fs.defaultFS","hdfs://pc-01:9000");//不配置默认本机 fs = FileSystem.get(conf); } //创建目录 @Test public void testMkdirs() throws IOException { fs.mkdirs(new Path("/bbb/zzz/")); fs.close(); } //删除文件 @Test public void testDelete() throws IOException { fs.delete(new Path("/bbb/zzz/"),true); fs.close(); } //移动 重命名 @Test public void testMv() throws IOException { fs.rename(new Path("/bbb/zxz/"),new Path("/zxz/")); //移动和重命名是一回事 fs.close(); } //上传文件 @Test public void testPut() throws IOException { fs.copyFromLocalFile(new Path("d:/a.jpg"),new Path("/zxz/")); fs.close(); } //下载文件 @Test public void testDownload() throws IOException { fs.copyToLocalFile(new Path("/zxz/"),new Path("d:/java")); fs.close(); } //递归查看目录内容 @Test public void testLs1() throws IOException { RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path("/"), true); while (files.hasNext()) { LocatedFileStatus next = files.next(); System.out.println(next); } fs.close(); } //查看当前目录内容 @Test public void testLs2() throws IOException { FileStatus[] status = fs.listStatus(new Path("/")); for (FileStatus fileStatus : status) { System.out.println(fileStatus.toString()); } fs.close(); } //读取文件内容 (wordcount) @Test public void testReaddata() throws IOException { HashMap<String, Integer> cotMap = new HashMap<>(); FSDataInputStream in = fs.open(new Path("/aaa1.txt")); BufferedReader br = new BufferedReader(new InputStreamReader(in)); String next; while ((next = br.readLine()) != null) { System.out.println(next); String[] split = next.split(" "); //按照空格分割单词 for (String s : split) { cotMap.put(s,cotMap.getOrDefault(s,0)+1); //单词和出现次数存入map } } for (Map.Entry<String, Integer> entry : cotMap.entrySet()) { System.out.println(entry.getKey()+"--->"+entry.getValue()); } in.close(); //将计数结果重新存入HDFS中 FSDataOutputStream append = fs.create(new Path("/result.txt")); BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(append)); for (Map.Entry<String, Integer> entry : cotMap.entrySet()) { bw.write(entry.getKey()+"--->"+entry.getValue()); bw.newLine(); } bw.flush(); bw.close(); fs.close(); } //向文件中追加内容 @Test public void testAppendData() throws IOException { FSDataOutputStream os = fs.append(new Path("/aaa1.txt")); os.write("这个男的已经21K了".getBytes()); os.close(); fs.close(); } }