项目说明：本项目基于maven jdk8
《POM.xml》
<?xml version="1.0" encoding="UTF-8"?>
<project>
<modelVersion>4.0.0</modelVersion>
<repositories>
<repository>
<id>central</id>
<name>Central Repository</name>
<url>https://repo.maven.apache.org/maven2</url>
<layout>default</layout>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
</repositories>
<pluginRepositories>
<pluginRepository>
<id>central</id>
<name>Central Repository</name>
<url>https://repo.maven.apache.org/maven2</url>
<layout>default</layout>
<snapshots>
<enabled>false</enabled>
</snapshots>
<releases>
<updatePolicy>never</updatePolicy>
</releases>
</pluginRepository>
</pluginRepositories>
<build>
<directory>${project.basedir}/target</directory>
<outputDirectory>${project.build.directory}/classes</outputDirectory>
<finalName>${project.artifactId}-${project.version}</finalName>
<testOutputDirectory>${project.build.directory}/test-classes</testOutputDirectory>
<sourceDirectory>${project.basedir}/src/main/java</sourceDirectory>
<scriptSourceDirectory>${project.basedir}/src/main/scripts</scriptSourceDirectory>
<testSourceDirectory>${project.basedir}/src/test/java</testSourceDirectory>
<resources>
<resource>
<directory>${project.basedir}/src/main/resources</directory>
</resource>
</resources>
<testResources>
<testResource>
<directory>${project.basedir}/src/test/resources</directory>
</testResource>
</testResources>
<pluginManagement>
<!-- NOTE: These plugins will be removed from future versions of the super POM -->
<!-- They are kept for the moment as they are very unlikely to conflict with lifecycle mappings (MNG-4453) -->
<plugins>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.3</version>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.2-beta-5</version>
</plugin>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<version>2.8</version>
</plugin>
<plugin>
<artifactId>maven-release-plugin</artifactId>
<version>2.5.3</version>
</plugin>
</plugins>
</pluginManagement>
</build>
<reporting>
<outputDirectory>${project.build.directory}/site</outputDirectory>
</reporting>
<profiles>
<!-- NOTE: The release profile will be removed from future versions of the super POM -->
<profile>
<id>release-profile</id>
<activation>
<property>
<name>performRelease</name>
<value>true</value>
</property>
</activation>
<build>
<plugins>
<plugin>
<inherited>true</inherited>
<artifactId>maven-source-plugin</artifactId>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<inherited>true</inherited>
<artifactId>maven-javadoc-plugin</artifactId>
<executions>
<execution>
<id>attach-javadocs</id>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<inherited>true</inherited>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
<updateReleaseInfo>true</updateReleaseInfo>
</configuration>
</plugin>
</plugins>
</build>
</profile>
</profiles>
</project> * Hello world!
*使用Java程序读取hadoop的存储上的文件
*/
public class HelloHDFS
{
public static void main(String[] args) throws IOException {
commonReadFun();
}
public static void commonReadFun() throws IOException {
//创建配置对象
Configuration conf = new Configuration();
//设置dfs的连接信息
conf.set("fs.defaultFS","hdfs://192.168.40.57:9000");
//通过连接信息 得到文件系统
FileSystem fileSystem = FileSystem.get(conf);
// //使用文件系统在hdfs的根目录下创建目录lanqiao 覆盖创建
// boolean success = fileSystem.mkdirs(new Path("/lanqiao"));
// System.out.println(success);
// //判断文件是否存在
// success =fileSystem.exists(new Path("/hello.txt"));
// System.out.println(success);
// //删除目录 参数一：文件路径 参数二：文件是否真正的从hdfs删除
// success = fileSystem.delete(new Path("/lanqiao"),true);
// System.out.println(success);
// //检查目录是否存在
// success = fileSystem.exists(new Path("/lanqiao"));
// System.out.println(success);
//上传文件到hdfs
/* FSDataOutputStream out = fileSystem.create(new Path("/test.data"),true);
FileInputStream in = new FileInputStream("d://test.log");
IOUtils.copyBytes(in,out,1024,true);*/
//获取指定目录下的文件列表
FileStatus[] fileStatus = fileSystem.listStatus(new Path("/"));
for (FileStatus fs :fileStatus){
System.out.println(fs.getPath());//文件路径
System.out.println(fs.getPermission());//文件的读写权限
System.out.println(fs.getReplication());//文件分几块
}
}
public static void firstReade() throws IOException {
/**
* 第一种方式
*/
//由于URL默认只支持http协议，而hadoop的HDFS使用的是HDFS协议，所以在这里设置URL，使其支持hdfs协议
URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
//使用URL 访问HDFS 使用协议为hdfs 此时的hello.txt存在于hadoop存储的根目录下
URL url = new URL("hdfs://master:9000/hello.txt");
//调用url的openStrem（）方法获取一个InputStrem
InputStream in = url.openStream();
//使用hadoop提供的IOUtils的copyBytes（输入流，输出流，缓冲区大小，流是否在使用完之后自动关闭）
IOUtils.copyBytes(in ,System.out,1024,true);
}
}
版权声明：本文内容由互联网用户自发贡献，该文观点仅代表作者本人。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容， 请联系我们举报，一经查实，本站将立刻删除。
发布者：全栈程序员-站长，转载请注明出处：https://javaforall.net/111278.html原文链接：https://javaforall.net
