09_java访问Hadoop的HDFS

项目说明：本项目基于maven jdk8
《POM.xml》

<?xml version="1.0" encoding="UTF-8"?>
<project>
  <modelVersion>4.0.0</modelVersion>

  <repositories>
    <repository>
      <id>central</id>
      <name>Central Repository</name>
      <url>https://repo.maven.apache.org/maven2</url>
      <layout>default</layout>
      <snapshots>
        <enabled>false</enabled>
      </snapshots>
    </repository>
  </repositories>

  <pluginRepositories>
    <pluginRepository>
      <id>central</id>
      <name>Central Repository</name>
      <url>https://repo.maven.apache.org/maven2</url>
      <layout>default</layout>
      <snapshots>
        <enabled>false</enabled>
      </snapshots>
      <releases>
        <updatePolicy>never</updatePolicy>
      </releases>
    </pluginRepository>
  </pluginRepositories>

  <build>
    <directory>${project.basedir}/target</directory>
    <outputDirectory>${project.build.directory}/classes</outputDirectory>
    <finalName>${project.artifactId}-${project.version}</finalName>
    <testOutputDirectory>${project.build.directory}/test-classes</testOutputDirectory>
    <sourceDirectory>${project.basedir}/src/main/java</sourceDirectory>
    <scriptSourceDirectory>${project.basedir}/src/main/scripts</scriptSourceDirectory>
    <testSourceDirectory>${project.basedir}/src/test/java</testSourceDirectory>
    <resources>
      <resource>
        <directory>${project.basedir}/src/main/resources</directory>
      </resource>
    </resources>
    <testResources>
      <testResource>
        <directory>${project.basedir}/src/test/resources</directory>
      </testResource>
    </testResources>
    <pluginManagement>
      <!-- NOTE: These plugins will be removed from future versions of the super POM -->
      <!-- They are kept for the moment as they are very unlikely to conflict with lifecycle mappings (MNG-4453) -->
      <plugins>
        <plugin>
          <artifactId>maven-antrun-plugin</artifactId>
          <version>1.3</version>
        </plugin>
        <plugin>
          <artifactId>maven-assembly-plugin</artifactId>
          <version>2.2-beta-5</version>
        </plugin>
        <plugin>
          <artifactId>maven-dependency-plugin</artifactId>
          <version>2.8</version>
        </plugin>
        <plugin>
          <artifactId>maven-release-plugin</artifactId>
          <version>2.5.3</version>
        </plugin>
      </plugins>
    </pluginManagement>
  </build>

  <reporting>
    <outputDirectory>${project.build.directory}/site</outputDirectory>
  </reporting>

  <profiles>
    <!-- NOTE: The release profile will be removed from future versions of the super POM -->
    <profile>
      <id>release-profile</id>

      <activation>
        <property>
          <name>performRelease</name>
          <value>true</value>
        </property>
      </activation>

      <build>
        <plugins>
          <plugin>
            <inherited>true</inherited>
            <artifactId>maven-source-plugin</artifactId>
            <executions>
              <execution>
                <id>attach-sources</id>
                <goals>
                  <goal>jar</goal>
                </goals>
              </execution>
            </executions>
          </plugin>
          <plugin>
            <inherited>true</inherited>
            <artifactId>maven-javadoc-plugin</artifactId>
            <executions>
              <execution>
                <id>attach-javadocs</id>
                <goals>
                  <goal>jar</goal>
                </goals>
              </execution>
            </executions>
          </plugin>
          <plugin>
            <inherited>true</inherited>
            <artifactId>maven-deploy-plugin</artifactId>
            <configuration>
              <updateReleaseInfo>true</updateReleaseInfo>
            </configuration>
          </plugin>
        </plugins>
      </build>
    </profile>
  </profiles>

</project>

 * Hello world!
 *使用Java程序读取hadoop的存储上的文件
 */
public class HelloHDFS
{

    public static void main(String[] args) throws IOException {
        commonReadFun();
    }
    public static  void   commonReadFun() throws IOException {
       //创建配置对象
        Configuration conf = new Configuration();
        //设置dfs的连接信息
        conf.set("fs.defaultFS","hdfs://192.168.40.57:9000");
       //通过连接信息 得到文件系统
        FileSystem fileSystem = FileSystem.get(conf);
//        //使用文件系统在hdfs的根目录下创建目录lanqiao   覆盖创建
//        boolean success = fileSystem.mkdirs(new Path("/lanqiao"));
//        System.out.println(success);
//        //判断文件是否存在
//        success =fileSystem.exists(new Path("/hello.txt"));
//        System.out.println(success);
//        //删除目录   参数一：文件路径   参数二：文件是否真正的从hdfs删除
//        success = fileSystem.delete(new Path("/lanqiao"),true);
//        System.out.println(success);
//        //检查目录是否存在
//       success =  fileSystem.exists(new Path("/lanqiao"));
//        System.out.println(success);
        //上传文件到hdfs
/*        FSDataOutputStream out =  fileSystem.create(new Path("/test.data"),true);
        FileInputStream in = new FileInputStream("d://test.log");
        IOUtils.copyBytes(in,out,1024,true);*/
        //获取指定目录下的文件列表
        FileStatus[] fileStatus = fileSystem.listStatus(new Path("/"));
        for (FileStatus fs :fileStatus){
            System.out.println(fs.getPath());//文件路径
            System.out.println(fs.getPermission());//文件的读写权限
            System.out.println(fs.getReplication());//文件分几块

        }


    }

    public static  void  firstReade() throws IOException {
        /**
         * 第一种方式
         */
        //由于URL默认只支持http协议，而hadoop的HDFS使用的是HDFS协议，所以在这里设置URL，使其支持hdfs协议
        URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
        //使用URL 访问HDFS 使用协议为hdfs   此时的hello.txt存在于hadoop存储的根目录下
        URL url = new URL("hdfs://master:9000/hello.txt");
        //调用url的openStrem（）方法获取一个InputStrem
        InputStream in = url.openStream();
        //使用hadoop提供的IOUtils的copyBytes（输入流，输出流，缓冲区大小，流是否在使用完之后自动关闭）
        IOUtils.copyBytes(in ,System.out,1024,true);
    }
}

发布者：全栈程序员-站长，转载请注明出处：https://javaforall.net/111278.html原文链接：https://javaforall.net